Build from within a git clone by default

Docker does not respect the .dockerignore file when building from a tar. Instead by default we'll make a local clone into a tmp directory and build from there. Subsequent builds will reset the clone to match the checkout. Compared to building directly in the repo, we'll have reproducible builds. Compared to using a git archive: 1. .dockerignore is respected 2. We'll have faster builds - docker can be smarter about caching the build context on subsequent builds from a directory To build from the repo directly, set the build context to "." in the config. If there are uncommitted changes, we'll warn about them either being included or ignored depending on whether we build from the clone.
2024-05-10 15:23:43 +01:00
parent 88947b6a7b
commit 307750ff70
15 changed files with 253 additions and 76 deletions
--- a/test/cli/build_test.rb
+++ b/test/cli/build_test.rb
@@ -9,32 +9,84 @@ class CliBuildTest < CliTestCase
  end

  test "push" do
+    with_build_directory do
+      Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+      hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" }
+
+      run_command("push", "--verbose").tap do |output|
+        assert_hook_ran "pre-build", output, **hook_variables
+        assert_match /Cloning repo into build directory/, output
+        assert_match /git -C #{Dir.tmpdir}\/kamal-clones\/app-#{pwd_sha} clone #{Dir.pwd}/, output
+        assert_match /docker --version && docker buildx version/, output
+        assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
+      end
+    end
+  end
+
+  test "push reseting clone" do
+    with_build_directory do
+      stub_setup
+      build_dir = "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}/kamal/"
+
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:docker, :buildx, :create, "--use", "--name", "kamal-app-multiarch")
+
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:mkdir, "-p", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}")
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd)
+        .raises(SSHKit::Command::Failed.new("fatal: destination path 'kamal' already exists and is not an empty directory"))
+        .then
+        .returns(true)
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:git, "-C", build_dir, :remote, "set-url", :origin, Dir.pwd)
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:git, "-C", build_dir, :fetch, :origin)
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:git, "-C", build_dir, :reset, "--hard", Kamal::Git.revision)
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:git, "-C", build_dir, :clean, "-fdx")
+
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with(:docker, :buildx, :build, "--push", "--platform", "linux/amd64,linux/arm64", "--builder", "kamal-app-multiarch", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".")
+
+      run_command("push", "--verbose").tap do |output|
+        assert_match /Cloning repo into build directory/, output
+        assert_match /Resetting local clone/, output
+      end
+    end
+  end
+
+  test "push without clone" do
    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
    hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" }

-    run_command("push", "--verbose").tap do |output|
+    run_command("push", "--verbose", fixture: :without_clone).tap do |output|
+      assert_no_match /Cloning repo into build directory/, output
      assert_hook_ran "pre-build", output, **hook_variables
      assert_match /docker --version && docker buildx version/, output
-      assert_match /git archive -tar HEAD | docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile - as .*@localhost/, output
+      assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile . as .*@localhost/, output
    end
  end

  test "push without builder" do
-    stub_setup
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, "--version", "&&", :docker, :buildx, "version")
+    with_build_directory do
+      stub_setup

-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :buildx, :create, "--use", "--name", "kamal-app-multiarch")
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with(:docker, "--version", "&&", :docker, :buildx, "version")

-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*args| p args[0..6]; args[0..6] == [ :git, :archive, "--format=tar", :HEAD, "|", :docker, :buildx ] }
-      .raises(SSHKit::Command::Failed.new("no builder"))
-      .then
-      .returns(true)
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with(:docker, :buildx, :create, "--use", "--name", "kamal-app-multiarch")

-    run_command("push").tap do |output|
-      assert_match /WARN Missing compatible builder, so creating a new one first/, output
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| args[0..1] == [ :docker, :buildx ] }
+        .raises(SSHKit::Command::Failed.new("no builder"))
+        .then
+        .returns(true)
+
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with { |*args| args.first.start_with?("git") }
+
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with(:mkdir, "-p", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}")
+
+      run_command("push").tap do |output|
+        assert_match /WARN Missing compatible builder, so creating a new one first/, output
+      end
    end
  end

@@ -118,4 +170,17 @@ class CliBuildTest < CliTestCase
      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
        .with { |*args| args[0..1] == [ :docker, :buildx ] }
    end
+
+    def with_build_directory
+      build_directory = File.join Dir.tmpdir, "kamal-clones", "app-#{pwd_sha}", "kamal"
+      FileUtils.mkdir_p build_directory
+      FileUtils.touch File.join build_directory, "Dockerfile"
+      yield
+    ensure
+      FileUtils.rm_rf build_directory
+    end
+
+    def pwd_sha
+      Digest::SHA256.hexdigest(Dir.pwd)[0..12]
+    end
 end
--- a/test/commands/builder_test.rb
+++ b/test/commands/builder_test.rb
@@ -9,7 +9,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "cache" => { "type" => "gha" } })
    assert_equal "multiarch", builder.name
    assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
      builder.push.join(" ")
  end

@@ -17,7 +17,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "multiarch" => false })
    assert_equal "native", builder.name
    assert_equal \
-      "git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
      builder.push.join(" ")
  end

@@ -25,7 +25,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "multiarch" => false, "cache" => { "type" => "gha" } })
    assert_equal "native/cached", builder.name
    assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
      builder.push.join(" ")
  end

@@ -33,7 +33,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "local" => {}, "remote" => {}, "cache" => { "type" => "gha" } })
    assert_equal "multiarch/remote", builder.name
    assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
      builder.push.join(" ")
  end

@@ -41,7 +41,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "local" => { "arch" => "amd64" } })
    assert_equal "multiarch", builder.name
    assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .",
      builder.push.join(" ")
  end

@@ -49,7 +49,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "remote" => { "arch" => "amd64" }, "cache" => { "type" => "gha" } })
    assert_equal "native/remote", builder.name
    assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64 --builder kamal-app-native-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64 --builder kamal-app-native-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
      builder.push.join(" ")
  end

@@ -100,21 +100,21 @@ class CommandsBuilderTest < ActiveSupport::TestCase
  test "native push with build args" do
    builder = new_builder_command(builder: { "multiarch" => false, "args" => { "a" => 1, "b" => 2 } })
    assert_equal \
-      "git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
      builder.push.join(" ")
  end

  test "multiarch push with build args" do
    builder = new_builder_command(builder: { "args" => { "a" => 1, "b" => 2 } })
    assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile .",
      builder.push.join(" ")
  end

  test "native push with build secrets" do
    builder = new_builder_command(builder: { "multiarch" => false, "secrets" => [ "a", "b" ] })
    assert_equal \
-      "git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
      builder.push.join(" ")
  end

@@ -162,4 +162,8 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    def new_builder_command(additional_config = {})
      Kamal::Commands::Builder.new(Kamal::Configuration.new(@config.merge(additional_config), version: "123"))
    end
+
+    def build_directory
+      "#{Dir.tmpdir}/kamal-clones/app/kamal/"
+    end
 end
--- a/test/configuration/builder_test.rb
+++ b/test/configuration/builder_test.rb
@@ -140,7 +140,7 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase
  end

  test "context" do
-    assert_equal "-", @config.builder.context
+    assert_equal ".", @config.builder.context
  end

  test "setting context" do
--- a/test/fixtures/deploy_without_clone.yml
+++ b/test/fixtures/deploy_without_clone.yml
@@ -0,0 +1,39 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+registry:
+  username: user
+  password: pw
+
+accessories:
+  mysql:
+    image: mysql:5.7
+    host: 1.1.1.3
+    port: 3306
+    env:
+      clear:
+        MYSQL_ROOT_HOST: '%'
+      secret:
+        - MYSQL_ROOT_PASSWORD
+    files:
+      - test/fixtures/files/my.cnf:/etc/mysql/my.cnf
+    directories:
+      - data:/var/lib/mysql
+  redis:
+    image: redis:latest
+    roles:
+      - web
+    port: 6379
+    directories:
+      - data:/data
+
+readiness_delay: 0
+
+builder:
+  context: "."