spacebar/kamal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build from within a git clone by default

Browse Source 
Docker does not respect the .dockerignore file when building from a tar.

Instead by default we'll make a local clone into a tmp directory and
build from there. Subsequent builds will reset the clone to match the
checkout.

Compared to building directly in the repo, we'll have reproducible
builds.

Compared to using a git archive:
1. .dockerignore is respected
2. We'll have faster builds - docker can be smarter about caching the
build context on subsequent builds from a directory

To build from the repo directly, set the build context to "." in the
config.

If there are uncommitted changes, we'll warn about them either being
included or ignored depending on whether we build from the clone.
This commit is contained in:
Donal McBreen
2024-05-10 15:23:43 +01:00
parent 88947b6a7b
commit 307750ff70
15 changed files with 253 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
lib/kamal/cli/build.rb
View File

@@ -19,21 +19,34 @@ class Kamal::Cli::Build < Kamal::Cli::Base
verify_local_dependencies
run_hook "pre-build"
if (uncommitted_changes = Kamal::Git.uncommitted_changes).present?
say "The following paths have uncommitted changes:\n #{uncommitted_changes}", :yellow
uncommitted_changes = Kamal::Git.uncommitted_changes
if KAMAL.config.builder.git_clone?
if uncommitted_changes.present?
say "Building from a local git clone, so ignoring these uncommitted changes:\n #{uncommitted_changes}", :yellow
end
prepare_clone
elsif uncommitted_changes.present?
say "Building with uncommitted changes:\n #{uncommitted_changes}", :yellow
end
# Get the command here to ensure the Dir.chdir doesn't interfere with it
push = KAMAL.builder.push
run_locally do
begin
KAMAL.with_verbosity(:debug) do
execute *KAMAL.builder.push
Dir.chdir(KAMAL.config.builder.build_directory) { execute *push }
end
rescue SSHKit::Command::Failed => e
if e.message =~ /(no builder)|(no such file or directory)/
warn "Missing compatible builder, so creating a new one first"
if cli.create
KAMAL.with_verbosity(:debug) { execute *KAMAL.builder.push }
KAMAL.with_verbosity(:debug) do
Dir.chdir(KAMAL.config.builder.build_directory) { execute *push }
end
end
else
raise
@@ -120,4 +133,23 @@ class Kamal::Cli::Build < Kamal::Cli::Base
end
end
end
def prepare_clone
run_locally do
begin
info "Cloning repo into build directory `#{KAMAL.config.builder.build_directory}`..."
execute *KAMAL.builder.create_clone_directory
execute *KAMAL.builder.clone
rescue SSHKit::Command::Failed => e
if e.message =~ /already exists and is not an empty directory/
info "Resetting local clone as `#{KAMAL.config.builder.build_directory}` already exists..."
KAMAL.builder.clone_reset_steps.each { |step| execute *step }
else
raise
end
end
end
end
end

4
lib/kamal/commands/base.rb
View File

@@ -78,8 +78,8 @@ module Kamal::Commands
args.compact.unshift :docker
end
def git(*args)
args.compact.unshift :git
def git(*args, path: nil)
[ :git, *([ "-C", path ] if path), *args.compact ]
end
def tags(**details)

18
lib/kamal/commands/builder.rb
View File

@@ -2,6 +2,7 @@ require "active_support/core_ext/string/filters"
class Kamal::Commands::Builder < Kamal::Commands::Base
delegate :create, :remove, :push, :clean, :pull, :info, :validate_image, to: :target
delegate :clone_directory, :build_directory, to: :"config.builder"
def name
target.class.to_s.remove("Kamal::Commands::Builder::").underscore.inquiry
@@ -53,6 +54,23 @@ class Kamal::Commands::Builder < Kamal::Commands::Base
end
end
def create_clone_directory
make_directory clone_directory
end
def clone
git :clone, Kamal::Git.root, path: clone_directory
end
def clone_reset_steps
[
git(:remote, "set-url", :origin, Kamal::Git.root, path: build_directory),
git(:fetch, :origin, path: build_directory),
git(:reset, "--hard", Kamal::Git.revision, path: build_directory),
git(:clean, "-fdx", path: build_directory)
]
end
private
def ensure_local_docker_installed
docker "--version"

12
lib/kamal/commands/builder/base.rb
View File

@@ -3,7 +3,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
class BuilderError < StandardError; end
delegate :argumentize, to: Kamal::Utils
delegate :args, :secrets, :dockerfile, :target, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, :ssh, :git_archive?, to: :builder_config
delegate :args, :secrets, :dockerfile, :target, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, :ssh, to: :builder_config
def clean
docker :image, :rm, "--force", config.absolute_image
@@ -13,16 +13,6 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
docker :pull, config.absolute_image
end
def push
if git_archive?
pipe \
git(:archive, "--format=tar", :HEAD),
build_and_push
else
build_and_push
end
end
def build_options
[ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_target, *build_ssh ]
end

18
lib/kamal/commands/builder/multiarch.rb
View File

@@ -13,6 +13,15 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
docker(:buildx, :ls)
end
def push
docker :buildx, :build,
"--push",
"--platform", platform_names,
"--builder", builder_name,
*build_options,
build_context
end
private
def builder_name
"kamal-#{config.service}-multiarch"
@@ -25,13 +34,4 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
"linux/amd64,linux/arm64"
end
end
def build_and_push
docker :buildx, :build,
"--push",
"--platform", platform_names,
"--builder", builder_name,
*build_options,
build_context
end
end

3
lib/kamal/commands/builder/native.rb
View File

@@ -11,8 +11,7 @@ class Kamal::Commands::Builder::Native < Kamal::Commands::Builder::Base
# No-op on native
end
private
def build_and_push
def push
combine \
docker(:build, *build_options, build_context),
docker(:push, config.absolute_image),

3
lib/kamal/commands/builder/native/cached.rb
View File

@@ -7,8 +7,7 @@ class Kamal::Commands::Builder::Native::Cached < Kamal::Commands::Builder::Nativ
docker :buildx, :rm, builder_name
end
private
def build_and_push
def push
docker :buildx, :build,
"--push",
*build_options,

18
lib/kamal/commands/builder/native/remote.rb
View File

@@ -17,6 +17,15 @@ class Kamal::Commands::Builder::Native::Remote < Kamal::Commands::Builder::Nativ
docker(:buildx, :ls)
end
def push
docker :buildx, :build,
"--push",
"--platform", platform,
"--builder", builder_name,
*build_options,
build_context
end
private
def builder_name
@@ -47,13 +56,4 @@ class Kamal::Commands::Builder::Native::Remote < Kamal::Commands::Builder::Nativ
def remove_buildx
docker :buildx, :rm, builder_name
end
def build_and_push
docker :buildx, :build,
"--push",
"--platform", platform,
"--builder", builder_name,
*build_options,
build_context
end
end

2
lib/kamal/configuration.rb
View File

@@ -336,7 +336,7 @@ class Kamal::Configuration
def git_version
@git_version ||=
if Kamal::Git.used?
if Kamal::Git.uncommitted_changes.present? && !builder.git_archive?
if Kamal::Git.uncommitted_changes.present? && !builder.git_clone?
uncommitted_suffix = "_uncommitted_#{SecureRandom.hex(8)}"
end
[ Kamal::Git.revision, uncommitted_suffix ].compact.join

31
lib/kamal/configuration/builder.rb
View File

@@ -3,6 +3,8 @@ class Kamal::Configuration::Builder
@options = config.raw_config.builder || {}
@image = config.image
@server = config.registry["server"]
@service = config.service
@destination = config.destination
valid?
end
@@ -44,7 +46,7 @@ class Kamal::Configuration::Builder
end
def context
@options["context"] || (git_archive? ? "-" : ".")
@options["context"] || "."
end
def local_arch
@@ -89,10 +91,23 @@ class Kamal::Configuration::Builder
@options["ssh"]
end
def git_archive?
def git_clone?
Kamal::Git.used? && @options["context"].nil?
end
def clone_directory
@clone_directory ||= File.join Dir.tmpdir, "kamal-clones", [ @service, @destination, pwd_sha ].compact.join("-")
end
def build_directory
@build_directory ||=
if git_clone?
File.join clone_directory, repo_basename, repo_relative_pwd
else
"."
end
end
private
def valid?
if @options["cache"] && @options["cache"]["type"]
@@ -123,4 +138,16 @@ class Kamal::Configuration::Builder
def cache_to_config_for_registry
[ "type=registry", @options["cache"]&.fetch("options", nil), "ref=#{cache_image_ref}" ].compact.join(",")
end
def repo_basename
File.basename(Kamal::Git.root)
end
def repo_relative_pwd
Dir.pwd.delete_prefix(Kamal::Git.root)
end
def pwd_sha
Digest::SHA256.hexdigest(Dir.pwd)[0..12]
end
end

4
lib/kamal/git.rb
View File

@@ -16,4 +16,8 @@ module Kamal::Git
def uncommitted_changes
`git status --porcelain`.strip
end
def root
`git rev-parse --show-toplevel`.strip
end
end

69
test/cli/build_test.rb
View File

@@ -9,18 +9,65 @@ class CliBuildTest < CliTestCase
end
test "push" do
with_build_directory do
Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" }
run_command("push", "--verbose").tap do |output|
assert_hook_ran "pre-build", output, **hook_variables
assert_match /Cloning repo into build directory/, output
assert_match /git -C #{Dir.tmpdir}\/kamal-clones\/app-#{pwd_sha} clone #{Dir.pwd}/, output
assert_match /docker --version && docker buildx version/, output
assert_match /git archive -tar HEAD | docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile - as .*@localhost/, output
assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
end
end
end
test "push reseting clone" do
with_build_directory do
stub_setup
build_dir = "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}/kamal/"
SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:docker, :buildx, :create, "--use", "--name", "kamal-app-multiarch")
SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:mkdir, "-p", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}")
SSHKit::Backend::Abstract.any_instance.stubs(:execute)
.with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd)
.raises(SSHKit::Command::Failed.new("fatal: destination path 'kamal' already exists and is not an empty directory"))
.then
.returns(true)
SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:git, "-C", build_dir, :remote, "set-url", :origin, Dir.pwd)
SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:git, "-C", build_dir, :fetch, :origin)
SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:git, "-C", build_dir, :reset, "--hard", Kamal::Git.revision)
SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:git, "-C", build_dir, :clean, "-fdx")
SSHKit::Backend::Abstract.any_instance.stubs(:execute)
.with(:docker, :buildx, :build, "--push", "--platform", "linux/amd64,linux/arm64", "--builder", "kamal-app-multiarch", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".")
run_command("push", "--verbose").tap do |output|
assert_match /Cloning repo into build directory/, output
assert_match /Resetting local clone/, output
end
end
end
test "push without clone" do
Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" }
run_command("push", "--verbose", fixture: :without_clone).tap do |output|
assert_no_match /Cloning repo into build directory/, output
assert_hook_ran "pre-build", output, **hook_variables
assert_match /docker --version && docker buildx version/, output
assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile . as .*@localhost/, output
end
end
test "push without builder" do
with_build_directory do
stub_setup
SSHKit::Backend::Abstract.any_instance.stubs(:execute)
.with(:docker, "--version", "&&", :docker, :buildx, "version")
@@ -28,15 +75,20 @@ class CliBuildTest < CliTestCase
.with(:docker, :buildx, :create, "--use", "--name", "kamal-app-multiarch")
SSHKit::Backend::Abstract.any_instance.stubs(:execute)
.with { |*args| p args[0..6]; args[0..6] == [ :git, :archive, "--format=tar", :HEAD, "|", :docker, :buildx ] }
.with { |*args| args[0..1] == [ :docker, :buildx ] }
.raises(SSHKit::Command::Failed.new("no builder"))
.then
.returns(true)
SSHKit::Backend::Abstract.any_instance.stubs(:execute).with { |*args| args.first.start_with?("git") }
SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:mkdir, "-p", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}")
run_command("push").tap do |output|
assert_match /WARN Missing compatible builder, so creating a new one first/, output
end
end
end
test "push with no buildx plugin" do
stub_setup
@@ -118,4 +170,17 @@ class CliBuildTest < CliTestCase
SSHKit::Backend::Abstract.any_instance.stubs(:execute)
.with { |*args| args[0..1] == [ :docker, :buildx ] }
end
def with_build_directory
build_directory = File.join Dir.tmpdir, "kamal-clones", "app-#{pwd_sha}", "kamal"
FileUtils.mkdir_p build_directory
FileUtils.touch File.join build_directory, "Dockerfile"
yield
ensure
FileUtils.rm_rf build_directory
end
def pwd_sha
Digest::SHA256.hexdigest(Dir.pwd)[0..12]
end
end

22
test/commands/builder_test.rb
View File

@@ -9,7 +9,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
builder = new_builder_command(builder: { "cache" => { "type" => "gha" } })
assert_equal "multiarch", builder.name
assert_equal \
"git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
"docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
builder.push.join(" ")
end
@@ -17,7 +17,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
builder = new_builder_command(builder: { "multiarch" => false })
assert_equal "native", builder.name
assert_equal \
"git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
"docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
builder.push.join(" ")
end
@@ -25,7 +25,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
builder = new_builder_command(builder: { "multiarch" => false, "cache" => { "type" => "gha" } })
assert_equal "native/cached", builder.name
assert_equal \
"git archive --format=tar HEAD | docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
"docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
builder.push.join(" ")
end
@@ -33,7 +33,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
builder = new_builder_command(builder: { "local" => {}, "remote" => {}, "cache" => { "type" => "gha" } })
assert_equal "multiarch/remote", builder.name
assert_equal \
"git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
"docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
builder.push.join(" ")
end
@@ -41,7 +41,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
builder = new_builder_command(builder: { "local" => { "arch" => "amd64" } })
assert_equal "multiarch", builder.name
assert_equal \
"git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile -",
"docker buildx build --push --platform linux/amd64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .",
builder.push.join(" ")
end
@@ -49,7 +49,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
builder = new_builder_command(builder: { "remote" => { "arch" => "amd64" }, "cache" => { "type" => "gha" } })
assert_equal "native/remote", builder.name
assert_equal \
"git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64 --builder kamal-app-native-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
"docker buildx build --push --platform linux/amd64 --builder kamal-app-native-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
builder.push.join(" ")
end
@@ -100,21 +100,21 @@ class CommandsBuilderTest < ActiveSupport::TestCase
test "native push with build args" do
builder = new_builder_command(builder: { "multiarch" => false, "args" => { "a" => 1, "b" => 2 } })
assert_equal \
"git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
"docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
builder.push.join(" ")
end
test "multiarch push with build args" do
builder = new_builder_command(builder: { "args" => { "a" => 1, "b" => 2 } })
assert_equal \
"git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile -",
"docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile .",
builder.push.join(" ")
end
test "native push with build secrets" do
builder = new_builder_command(builder: { "multiarch" => false, "secrets" => [ "a", "b" ] })
assert_equal \
"git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
"docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
builder.push.join(" ")
end
@@ -162,4 +162,8 @@ class CommandsBuilderTest < ActiveSupport::TestCase
def new_builder_command(additional_config = {})
Kamal::Commands::Builder.new(Kamal::Configuration.new(@config.merge(additional_config), version: "123"))
end
def build_directory
"#{Dir.tmpdir}/kamal-clones/app/kamal/"
end
end

2
test/configuration/builder_test.rb
View File

@@ -140,7 +140,7 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase
end
test "context" do
assert_equal "-", @config.builder.context
assert_equal ".", @config.builder.context
end
test "setting context" do

39
test/fixtures/deploy_without_clone.yml vendored Normal file
View File

@@ -0,0 +1,39 @@
service: app
image: dhh/app
servers:
web:
- "1.1.1.1"
- "1.1.1.2"
workers:
- "1.1.1.3"
- "1.1.1.4"
registry:
username: user
password: pw
accessories:
mysql:
image: mysql:5.7
host: 1.1.1.3
port: 3306
env:
clear:
MYSQL_ROOT_HOST: '%'
secret:
- MYSQL_ROOT_PASSWORD
files:
- test/fixtures/files/my.cnf:/etc/mysql/my.cnf
directories:
- data:/var/lib/mysql
redis:
image: redis:latest
roles:
- web
port: 6379
directories:
- data:/data
readiness_delay: 0
builder:
context: "."