Bump version for 1.6.0

Merge pull request #821 from basecamp/retry-clone
Handle corrupt git clones
2024-06-03 08:34:12 +01:00 · 2024-05-28 15:23:36 +01:00 · 2024-05-27 11:17:34 +01:00 · 2024-05-21 12:33:45 +01:00 · 2024-05-21 12:20:19 +01:00 · 2024-05-21 12:02:16 +01:00
65 changed files with 1369 additions and 677 deletions
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
  remote: .
  specs:
-    kamal (1.5.2)
+    kamal (1.6.0)
      activesupport (>= 7.0)
      base64 (~> 0.2)
      bcrypt_pbkdf (~> 1.0)
@@ -9,7 +9,7 @@ PATH
      dotenv (~> 2.8)
      ed25519 (~> 1.2)
      net-ssh (~> 7.0)
-      sshkit (~> 1.21)
+      sshkit (>= 1.22.2, < 2.0)
      thor (~> 1.2)
      zeitwerk (~> 2.5)

@@ -75,6 +75,8 @@ GEM
    mutex_m (0.2.0)
    net-scp (4.0.0)
      net-ssh (>= 2.6.5, < 8.0.0)
+    net-sftp (4.0.0)
+      net-ssh (>= 5.0.0, < 8.0.0)
    net-ssh (7.2.1)
    nokogiri (1.16.0-arm64-darwin)
      racc (~> 1.4)
@@ -151,9 +153,11 @@ GEM
      rubocop-rails
    ruby-progressbar (1.13.0)
    ruby2_keywords (0.0.5)
-    sshkit (1.21.7)
+    sshkit (1.22.2)
+      base64
      mutex_m
      net-scp (>= 1.1.2)
+      net-sftp (>= 2.1.2)
      net-ssh (>= 2.8.0)
    stringio (3.1.0)
    thor (1.3.0)
--- a/kamal.gemspec
+++ b/kamal.gemspec
@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
  spec.executables = %w[ kamal ]

  spec.add_dependency "activesupport", ">= 7.0"
-  spec.add_dependency "sshkit", "~> 1.21"
+  spec.add_dependency "sshkit", ">= 1.22.2", "< 2.0"
  spec.add_dependency "net-ssh", "~> 7.0"
  spec.add_dependency "thor", "~> 1.2"
  spec.add_dependency "dotenv", "~> 2.8"
--- a/lib/kamal/cli/accessory.rb
+++ b/lib/kamal/cli/accessory.rb
@@ -1,7 +1,7 @@
 class Kamal::Cli::Accessory < Kamal::Cli::Base
  desc "boot [NAME]", "Boot new accessory service on host (use NAME=all to boot all accessories)"
  def boot(name, login: true)
-    mutating do
+    with_lock do
      if name == "all"
        KAMAL.accessory_names.each { |accessory_name| boot(accessory_name) }
      else
@@ -21,7 +21,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base

  desc "upload [NAME]", "Upload accessory files to host", hide: true
  def upload(name)
-    mutating do
+    with_lock do
      with_accessory(name) do |accessory, hosts|
        on(hosts) do
          accessory.files.each do |(local, remote)|
@@ -38,7 +38,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base

  desc "directories [NAME]", "Create accessory directories on host", hide: true
  def directories(name)
-    mutating do
+    with_lock do
      with_accessory(name) do |accessory, hosts|
        on(hosts) do
          accessory.directories.keys.each do |host_path|
@@ -51,7 +51,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base

  desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container; use NAME=all to boot all accessories)"
  def reboot(name)
-    mutating do
+    with_lock do
      if name == "all"
        KAMAL.accessory_names.each { |accessory_name| reboot(accessory_name) }
      else
@@ -70,7 +70,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base

  desc "start [NAME]", "Start existing accessory container on host"
  def start(name)
-    mutating do
+    with_lock do
      with_accessory(name) do |accessory, hosts|
        on(hosts) do
          execute *KAMAL.auditor.record("Started #{name} accessory"), verbosity: :debug
@@ -82,7 +82,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base

  desc "stop [NAME]", "Stop existing accessory container on host"
  def stop(name)
-    mutating do
+    with_lock do
      with_accessory(name) do |accessory, hosts|
        on(hosts) do
          execute *KAMAL.auditor.record("Stopped #{name} accessory"), verbosity: :debug
@@ -94,7 +94,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base

  desc "restart [NAME]", "Restart existing accessory container on host"
  def restart(name)
-    mutating do
+    with_lock do
      with_accessory(name) do
        stop(name)
        start(name)
@@ -174,17 +174,12 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
  desc "remove [NAME]", "Remove accessory container, image and data directory from host (use NAME=all to remove all accessories)"
  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
  def remove(name)
-    mutating do
-      if name == "all"
-        KAMAL.accessory_names.each { |accessory_name| remove(accessory_name) }
-      else
-        confirming "This will remove all containers, images and data directories for #{name}. Are you sure?" do
-          with_accessory(name) do
-            stop(name)
-            remove_container(name)
-            remove_image(name)
-            remove_service_directory(name)
-          end
+    confirming "This will remove all containers, images and data directories for #{name}. Are you sure?" do
+      with_lock do
+        if name == "all"
+          KAMAL.accessory_names.each { |accessory_name| remove_accessory(accessory_name) }
+        else
+          remove_accessory(name)
        end
      end
    end
@@ -192,7 +187,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base

  desc "remove_container [NAME]", "Remove accessory container from host", hide: true
  def remove_container(name)
-    mutating do
+    with_lock do
      with_accessory(name) do |accessory, hosts|
        on(hosts) do
          execute *KAMAL.auditor.record("Remove #{name} accessory container"), verbosity: :debug
@@ -204,7 +199,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base

  desc "remove_image [NAME]", "Remove accessory image from host", hide: true
  def remove_image(name)
-    mutating do
+    with_lock do
      with_accessory(name) do |accessory, hosts|
        on(hosts) do
          execute *KAMAL.auditor.record("Removed #{name} accessory image"), verbosity: :debug
@@ -216,7 +211,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base

  desc "remove_service_directory [NAME]", "Remove accessory directory used for uploaded files and data directories from host", hide: true
  def remove_service_directory(name)
-    mutating do
+    with_lock do
      with_accessory(name) do |accessory, hosts|
        on(hosts) do
          execute *accessory.remove_service_directory
@@ -250,4 +245,13 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
        accessory.hosts
      end
    end
+
+    def remove_accessory(name)
+      with_accessory(name) do
+        stop(name)
+        remove_container(name)
+        remove_image(name)
+        remove_service_directory(name)
+      end
+    end
 end
--- a/lib/kamal/cli/app.rb
+++ b/lib/kamal/cli/app.rb
@@ -1,43 +1,45 @@
 class Kamal::Cli::App < Kamal::Cli::Base
  desc "boot", "Boot app on servers (or reboot app if already running)"
  def boot
-    mutating do
-      hold_lock_on_error do
-        say "Get most recent version available as an image...", :magenta unless options[:version]
-        using_version(version_or_latest) do |version|
-          say "Start container with version #{version} using a #{KAMAL.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
+    with_lock do
+      say "Get most recent version available as an image...", :magenta unless options[:version]
+      using_version(version_or_latest) do |version|
+        say "Start container with version #{version} using a #{KAMAL.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta

-          # Assets are prepared in a separate step to ensure they are on all hosts before booting
-          on(KAMAL.hosts) do
-            KAMAL.roles_on(host).each do |role|
-              Kamal::Cli::App::PrepareAssets.new(host, role, self).run
-            end
+        # Assets are prepared in a separate step to ensure they are on all hosts before booting
+        on(KAMAL.hosts) do
+          KAMAL.roles_on(host).each do |role|
+            Kamal::Cli::App::PrepareAssets.new(host, role, self).run
          end
+        end

-          on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
-            KAMAL.roles_on(host).each do |role|
-              Kamal::Cli::App::Boot.new(host, role, version, self).run
-            end
-          end
+        # Primary hosts and roles are returned first, so they can open the barrier
+        barrier = Kamal::Cli::Healthcheck::Barrier.new if KAMAL.roles.many?

-          on(KAMAL.hosts) do |host|
-            execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
-            execute *KAMAL.app.tag_latest_image
+        on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
+          KAMAL.roles_on(host).each do |role|
+            Kamal::Cli::App::Boot.new(host, role, self, version, barrier).run
          end
        end
+
+        # Tag once the app booted on all hosts
+        on(KAMAL.hosts) do |host|
+          execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
+          execute *KAMAL.app.tag_latest_image
+        end
      end
    end
  end

  desc "start", "Start existing app container on servers"
  def start
-    mutating do
+    with_lock do
      on(KAMAL.hosts) do |host|
        roles = KAMAL.roles_on(host)

        roles.each do |role|
          execute *KAMAL.auditor.record("Started app version #{KAMAL.config.version}"), verbosity: :debug
-          execute *KAMAL.app(role: role).start, raise_on_non_zero_exit: false
+          execute *KAMAL.app(role: role, host: host).start, raise_on_non_zero_exit: false
        end
      end
    end
@@ -45,13 +47,13 @@ class Kamal::Cli::App < Kamal::Cli::Base

  desc "stop", "Stop app container on servers"
  def stop
-    mutating do
+    with_lock do
      on(KAMAL.hosts) do |host|
        roles = KAMAL.roles_on(host)

        roles.each do |role|
          execute *KAMAL.auditor.record("Stopped app", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).stop, raise_on_non_zero_exit: false
+          execute *KAMAL.app(role: role, host: host).stop, raise_on_non_zero_exit: false
        end
      end
    end
@@ -64,12 +66,12 @@ class Kamal::Cli::App < Kamal::Cli::Base
      roles = KAMAL.roles_on(host)

      roles.each do |role|
-        puts_by_host host, capture_with_info(*KAMAL.app(role: role).info)
+        puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).info)
      end
    end
  end

-  desc "exec [CMD]", "Execute a custom command on servers (use --help to show options)"
+  desc "exec [CMD]", "Execute a custom command on servers within the app container (use --help to show options)"
  option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
  option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
  option :env, aliases: "-e", type: :hash, desc: "Set environment variables for the command"
@@ -80,7 +82,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
      say "Get current version of running container...", :magenta unless options[:version]
      using_version(options[:version] || current_running_version) do |version|
        say "Launching interactive command with version #{version} via SSH from existing container on #{KAMAL.primary_host}...", :magenta
-        run_locally { exec KAMAL.app(role: KAMAL.primary_role).execute_in_existing_container_over_ssh(cmd, host: KAMAL.primary_host, env: env) }
+        run_locally { exec KAMAL.app(role: KAMAL.primary_role, host: KAMAL.primary_host).execute_in_existing_container_over_ssh(cmd, env: env) }
      end

    when options[:interactive]
@@ -88,7 +90,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
      using_version(version_or_latest) do |version|
        say "Launching interactive command with version #{version} via SSH from new container on #{KAMAL.primary_host}...", :magenta
        run_locally do
-          exec KAMAL.app(role: KAMAL.primary_role).execute_in_new_container_over_ssh(cmd, host: KAMAL.primary_host, env: env)
+          exec KAMAL.app(role: KAMAL.primary_role, host: KAMAL.primary_host).execute_in_new_container_over_ssh(cmd, env: env)
        end
      end

@@ -102,7 +104,7 @@ class Kamal::Cli::App < Kamal::Cli::Base

          roles.each do |role|
            execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}", role: role), verbosity: :debug
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_existing_container(cmd, env: env))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).execute_in_existing_container(cmd, env: env))
          end
        end
      end
@@ -116,7 +118,7 @@ class Kamal::Cli::App < Kamal::Cli::Base

          roles.each do |role|
            execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_new_container(cmd, env: env))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).execute_in_new_container(cmd, env: env))
          end
        end
      end
@@ -131,22 +133,21 @@ class Kamal::Cli::App < Kamal::Cli::Base
  desc "stale_containers", "Detect app stale containers"
  option :stop, aliases: "-s", type: :boolean, default: false, desc: "Stop the stale containers found"
  def stale_containers
-    mutating do
-      stop = options[:stop]
-
-      cli = self
+    stop = options[:stop]

+    with_lock_if_stopping do
      on(KAMAL.hosts) do |host|
        roles = KAMAL.roles_on(host)

        roles.each do |role|
-          versions = capture_with_info(*KAMAL.app(role: role).list_versions, raise_on_non_zero_exit: false).split("\n")
-          versions -= [ capture_with_info(*KAMAL.app(role: role).current_running_version, raise_on_non_zero_exit: false).strip ]
+          app = KAMAL.app(role: role, host: host)
+          versions = capture_with_info(*app.list_versions, raise_on_non_zero_exit: false).split("\n")
+          versions -= [ capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip ]

          versions.each do |version|
            if stop
              puts_by_host host, "Stopping stale container for role #{role} with version #{version}"
-              execute *KAMAL.app(role: role).stop(version: version), raise_on_non_zero_exit: false
+              execute *app.stop(version: version), raise_on_non_zero_exit: false
            else
              puts_by_host host,  "Detected stale container for role #{role} with version #{version} (use `kamal app stale_containers --stop` to stop)"
            end
@@ -180,8 +181,9 @@ class Kamal::Cli::App < Kamal::Cli::Base
        KAMAL.specific_roles ||= [ "web" ]
        role = KAMAL.roles_on(KAMAL.primary_host).first

-        info KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
-        exec KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
+        app = KAMAL.app(role: role, host: host)
+        info app.follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
+        exec app.follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
      end
    else
      lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
@@ -191,7 +193,7 @@ class Kamal::Cli::App < Kamal::Cli::Base

        roles.each do |role|
          begin
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).logs(since: since, lines: lines, grep: grep))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).logs(since: since, lines: lines, grep: grep))
          rescue SSHKit::Command::Failed
            puts_by_host host, "Nothing found"
          end
@@ -202,7 +204,7 @@ class Kamal::Cli::App < Kamal::Cli::Base

  desc "remove", "Remove app containers and images from servers"
  def remove
-    mutating do
+    with_lock do
      stop
      remove_containers
      remove_images
@@ -211,13 +213,13 @@ class Kamal::Cli::App < Kamal::Cli::Base

  desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true
  def remove_container(version)
-    mutating do
+    with_lock do
      on(KAMAL.hosts) do |host|
        roles = KAMAL.roles_on(host)

        roles.each do |role|
          execute *KAMAL.auditor.record("Removed app container with version #{version}", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).remove_container(version: version)
+          execute *KAMAL.app(role: role, host: host).remove_container(version: version)
        end
      end
    end
@@ -225,13 +227,13 @@ class Kamal::Cli::App < Kamal::Cli::Base

  desc "remove_containers", "Remove all app containers from servers", hide: true
  def remove_containers
-    mutating do
+    with_lock do
      on(KAMAL.hosts) do |host|
        roles = KAMAL.roles_on(host)

        roles.each do |role|
          execute *KAMAL.auditor.record("Removed all app containers", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).remove_containers
+          execute *KAMAL.app(role: role, host: host).remove_containers
        end
      end
    end
@@ -239,7 +241,7 @@ class Kamal::Cli::App < Kamal::Cli::Base

  desc "remove_images", "Remove all app images from servers", hide: true
  def remove_images
-    mutating do
+    with_lock do
      on(KAMAL.hosts) do
        execute *KAMAL.auditor.record("Removed all app images"), verbosity: :debug
        execute *KAMAL.app.remove_images
@@ -251,7 +253,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
  def version
    on(KAMAL.hosts) do |host|
      role = KAMAL.roles_on(host).first
-      puts_by_host host, capture_with_info(*KAMAL.app(role: role).current_running_version).strip
+      puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).current_running_version).strip
    end
  end

@@ -274,7 +276,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
      version = nil
      on(host) do
        role = KAMAL.roles_on(host).first
-        version = capture_with_info(*KAMAL.app(role: role).current_running_version).strip
+        version = capture_with_info(*KAMAL.app(role: role, host: host).current_running_version).strip
      end
      version.presence
    end
@@ -282,4 +284,12 @@ class Kamal::Cli::App < Kamal::Cli::Base
    def version_or_latest
      options[:version] || KAMAL.config.latest_tag
    end
+
+    def with_lock_if_stopping
+      if options[:stop]
+        with_lock { yield }
+      else
+        yield
+      end
+    end
 end
--- a/lib/kamal/cli/app/boot.rb
+++ b/lib/kamal/cli/app/boot.rb
@@ -1,19 +1,30 @@
 class Kamal::Cli::App::Boot
-  attr_reader :host, :role, :version, :sshkit
-  delegate :execute, :capture_with_info, :info, to: :sshkit
-  delegate :uses_cord?, :assets?, to: :role
+  attr_reader :host, :role, :version, :barrier, :sshkit
+  delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, to: :sshkit
+  delegate :uses_cord?, :assets?, :running_traefik?, to: :role

-  def initialize(host, role, version, sshkit)
+  def initialize(host, role, sshkit, version, barrier)
    @host = host
    @role = role
    @version = version
+    @barrier = barrier
    @sshkit = sshkit
  end

  def run
    old_version = old_version_renamed_if_clashing

-    start_new_version
+    wait_at_barrier if queuer?
+
+    begin
+      start_new_version
+    rescue => e
+      close_barrier if gatekeeper?
+      stop_new_version
+      raise
+    end
+
+    release_barrier if gatekeeper?

    if old_version
      stop_old_version(old_version)
@@ -21,18 +32,6 @@ class Kamal::Cli::App::Boot
  end

  private
-    def app
-      @app ||= KAMAL.app(role: role)
-    end
-
-    def auditor
-      @auditor = KAMAL.auditor(role: role)
-    end
-
-    def audit(message)
-      execute *auditor.record(message), verbosity: :debug
-    end
-
    def old_version_renamed_if_clashing
      if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
        renamed_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
@@ -46,11 +45,17 @@ class Kamal::Cli::App::Boot

    def start_new_version
      audit "Booted app version #{version}"
+
      execute *app.tie_cord(role.cord_host_file) if uses_cord?
-      execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
+      hostname = "#{host.to_s[0...51].gsub(/\.+$/, '')}-#{SecureRandom.hex(6)}"
+      execute *app.run(hostname: hostname)
      Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
    end

+    def stop_new_version
+      execute *app.stop(version: version), raise_on_non_zero_exit: false
+    end
+
    def stop_old_version(version)
      if uses_cord?
        cord = capture_with_info(*app.cord(version: version), raise_on_non_zero_exit: false).strip
@@ -64,4 +69,51 @@ class Kamal::Cli::App::Boot

      execute *app.clean_up_assets if assets?
    end
+
+    def release_barrier
+      if barrier.open
+        info "First #{KAMAL.primary_role} container is healthy on #{host}, booting other roles"
+      end
+    end
+
+    def wait_at_barrier
+      info "Waiting for the first healthy #{KAMAL.primary_role} container before booting #{role} on #{host}..."
+      barrier.wait
+      info "First #{KAMAL.primary_role} container is healthy, booting #{role} on #{host}..."
+    rescue Kamal::Cli::Healthcheck::Error
+      info "First #{KAMAL.primary_role} container is unhealthy, not booting #{role} on #{host}"
+      raise
+    end
+
+    def close_barrier
+      if barrier.close
+        info "First #{KAMAL.primary_role} container is unhealthy on #{host}, not booting other roles"
+        error capture_with_info(*app.logs(version: version))
+        error capture_with_info(*app.container_health_log(version: version))
+      end
+    end
+
+    def barrier_role?
+      role == KAMAL.primary_role
+    end
+
+    def app
+      @app ||= KAMAL.app(role: role, host: host)
+    end
+
+    def auditor
+      @auditor = KAMAL.auditor(role: role)
+    end
+
+    def audit(message)
+      execute *auditor.record(message), verbosity: :debug
+    end
+
+    def gatekeeper?
+      barrier && barrier_role?
+    end
+
+    def queuer?
+      barrier && !barrier_role?
+    end
 end
--- a/lib/kamal/cli/app/prepare_assets.rb
+++ b/lib/kamal/cli/app/prepare_assets.rb
@@ -19,6 +19,6 @@ class Kamal::Cli::App::PrepareAssets

  private
    def app
-      @app ||= KAMAL.app(role: role)
+      @app ||= KAMAL.app(role: role, host: host)
    end
 end
--- a/lib/kamal/cli/base.rb
+++ b/lib/kamal/cli/base.rb
@@ -79,28 +79,27 @@ module Kamal::Cli
        puts "  Finished all in #{sprintf("%.1f seconds", runtime)}"
      end

-      def mutating
-        return yield if KAMAL.holding_lock?
-
-        run_hook "pre-connect"
-
-        ensure_run_and_locks_directory
-
-        acquire_lock
-
-        begin
+      def with_lock
+        if KAMAL.holding_lock?
          yield
-        rescue
-          if KAMAL.hold_lock_on_error?
-            error "  \e[31mDeploy lock was not released\e[0m"
-          else
-            release_lock
+        else
+          ensure_run_and_locks_directory
+
+          acquire_lock
+
+          begin
+            yield
+          rescue
+            begin
+              release_lock
+            rescue => e
+              say "Error releasing the deploy lock: #{e.message}", :red
+            end
+            raise
          end

-          raise
+          release_lock
        end
-
-        release_lock
      end

      def confirming(question)
@@ -141,16 +140,6 @@ module Kamal::Cli
        end
      end

-      def hold_lock_on_error
-        if KAMAL.hold_lock_on_error?
-          yield
-        else
-          KAMAL.hold_lock_on_error = true
-          yield
-          KAMAL.hold_lock_on_error = false
-        end
-      end
-
      def run_hook(hook, **extra_details)
        if !options[:skip_hooks] && KAMAL.hook.hook_exists?(hook)
          details = { hosts: KAMAL.hosts.join(","), command: command, subcommand: subcommand }
@@ -164,6 +153,15 @@ module Kamal::Cli
        end
      end

+      def on(*args, &block)
+        if !KAMAL.connected?
+          run_hook "pre-connect"
+          KAMAL.connected = true
+        end
+
+        super
+      end
+
      def command
        @kamal_command ||= begin
          invocation_class, invocation_commands = *first_invocation
--- a/lib/kamal/cli/build.rb
+++ b/lib/kamal/cli/build.rb
@@ -5,39 +5,50 @@ class Kamal::Cli::Build < Kamal::Cli::Base

  desc "deliver", "Build app and push app image to registry then pull image on servers"
  def deliver
-    mutating do
-      push
-      pull
-    end
+    push
+    pull
  end

  desc "push", "Build and push app image to registry"
  def push
-    mutating do
-      cli = self
+    cli = self

-      verify_local_dependencies
-      run_hook "pre-build"
+    verify_local_dependencies
+    run_hook "pre-build"

-      if (uncommitted_changes = Kamal::Git.uncommitted_changes).present?
-        say "The following paths have uncommitted changes:\n #{uncommitted_changes}", :yellow
+    uncommitted_changes = Kamal::Git.uncommitted_changes
+
+    if KAMAL.config.builder.git_clone?
+      if uncommitted_changes.present?
+        say "Building from a local git clone, so ignoring these uncommitted changes:\n #{uncommitted_changes}", :yellow
      end

      run_locally do
-        begin
-          KAMAL.with_verbosity(:debug) do
-            execute *KAMAL.builder.push
-          end
-        rescue SSHKit::Command::Failed => e
-          if e.message =~ /(no builder)|(no such file or directory)/
-            warn "Missing compatible builder, so creating a new one first"
+        Clone.new(self).prepare
+      end
+    elsif uncommitted_changes.present?
+      say "Building with uncommitted changes:\n #{uncommitted_changes}", :yellow
+    end

-            if cli.create
-              KAMAL.with_verbosity(:debug) { execute *KAMAL.builder.push }
+    # Get the command here to ensure the Dir.chdir doesn't interfere with it
+    push = KAMAL.builder.push
+
+    run_locally do
+      begin
+        KAMAL.with_verbosity(:debug) do
+          Dir.chdir(KAMAL.config.builder.build_directory) { execute *push }
+        end
+      rescue SSHKit::Command::Failed => e
+        if e.message =~ /(no builder)|(no such file or directory)/
+          warn "Missing compatible builder, so creating a new one first"
+
+          if cli.create
+            KAMAL.with_verbosity(:debug) do
+              Dir.chdir(KAMAL.config.builder.build_directory) { execute *push }
            end
-          else
-            raise
          end
+        else
+          raise
        end
      end
    end
@@ -45,34 +56,30 @@ class Kamal::Cli::Build < Kamal::Cli::Base

  desc "pull", "Pull app image from registry onto servers"
  def pull
-    mutating do
-      on(KAMAL.hosts) do
-        execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
-        execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
-        execute *KAMAL.builder.pull
-        execute *KAMAL.builder.validate_image
-      end
+    on(KAMAL.hosts) do
+      execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
+      execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
+      execute *KAMAL.builder.pull
+      execute *KAMAL.builder.validate_image
    end
  end

  desc "create", "Create a build setup"
  def create
-    mutating do
-      if (remote_host = KAMAL.config.builder.remote_host)
-        connect_to_remote_host(remote_host)
-      end
+    if (remote_host = KAMAL.config.builder.remote_host)
+      connect_to_remote_host(remote_host)
+    end

-      run_locally do
-        begin
-          debug "Using builder: #{KAMAL.builder.name}"
-          execute *KAMAL.builder.create
-        rescue SSHKit::Command::Failed => e
-          if e.message =~ /stderr=(.*)/
-            error "Couldn't create remote builder: #{$1}"
-            false
-          else
-            raise
-          end
+    run_locally do
+      begin
+        debug "Using builder: #{KAMAL.builder.name}"
+        execute *KAMAL.builder.create
+      rescue SSHKit::Command::Failed => e
+        if e.message =~ /stderr=(.*)/
+          error "Couldn't create remote builder: #{$1}"
+          false
+        else
+          raise
        end
      end
    end
@@ -80,11 +87,9 @@ class Kamal::Cli::Build < Kamal::Cli::Base

  desc "remove", "Remove build setup"
  def remove
-    mutating do
-      run_locally do
-        debug "Using builder: #{KAMAL.builder.name}"
-        execute *KAMAL.builder.remove
-      end
+    run_locally do
+      debug "Using builder: #{KAMAL.builder.name}"
+      execute *KAMAL.builder.remove
    end
  end

@@ -114,8 +119,11 @@ class Kamal::Cli::Build < Kamal::Cli::Base
    def connect_to_remote_host(remote_host)
      remote_uri = URI.parse(remote_host)
      if remote_uri.scheme == "ssh"
-        options = { user: remote_uri.user, port: remote_uri.port }.compact
-        on(remote_uri.host, options) do
+        host = SSHKit::Host.new(
+          hostname: remote_uri.host,
+          ssh_options: { user: remote_uri.user, port: remote_uri.port }.compact
+        )
+        on(host, options) do
          execute "true"
        end
      end
--- a/lib/kamal/cli/build/clone.rb
+++ b/lib/kamal/cli/build/clone.rb
@@ -0,0 +1,61 @@
+require "uri"
+
+class Kamal::Cli::Build::Clone
+  attr_reader :sshkit
+  delegate :info, :error, :execute, :capture_with_info, to: :sshkit
+
+  def initialize(sshkit)
+    @sshkit = sshkit
+  end
+
+  def prepare
+    begin
+      clone_repo
+    rescue SSHKit::Command::Failed => e
+      if e.message =~ /already exists and is not an empty directory/
+        reset
+      else
+        raise Kamal::Cli::Build::BuildError, "Failed to clone repo: #{e.message}"
+      end
+    end
+
+    validate!
+  rescue Kamal::Cli::Build::BuildError => e
+    error "Error preparing clone: #{e.message}, deleting and retrying..."
+
+    FileUtils.rm_rf KAMAL.config.builder.clone_directory
+    clone_repo
+    validate!
+  end
+
+  private
+    def clone_repo
+      info "Cloning repo into build directory `#{KAMAL.config.builder.build_directory}`..."
+
+      FileUtils.mkdir_p KAMAL.config.builder.clone_directory
+      execute *KAMAL.builder.clone
+    end
+
+    def reset
+      info "Resetting local clone as `#{KAMAL.config.builder.build_directory}` already exists..."
+
+      KAMAL.builder.clone_reset_steps.each { |step| execute *step }
+    rescue SSHKit::Command::Failed => e
+      raise Kamal::Cli::Build::BuildError, "Failed to clone repo: #{e.message}"
+    end
+
+    def validate!
+      status = capture_with_info(*KAMAL.builder.clone_status).strip
+
+      unless status.empty?
+        raise Kamal::Cli::Build::BuildError, "Clone in #{KAMAL.config.builder.build_directory} is dirty, #{status}"
+      end
+
+      revision = capture_with_info(*KAMAL.builder.clone_revision).strip
+      if revision != Kamal::Git.revision
+        raise Kamal::Cli::Build::BuildError, "Clone in #{KAMAL.config.builder.build_directory} is not on the correct revision, expected `#{Kamal::Git.revision}` but got `#{revision}`"
+      end
+    rescue SSHKit::Command::Failed => e
+      raise Kamal::Cli::Build::BuildError, "Failed to validate clone: #{e.message}"
+    end
+end
--- a/lib/kamal/cli/env.rb
+++ b/lib/kamal/cli/env.rb
@@ -3,13 +3,13 @@ require "tempfile"
 class Kamal::Cli::Env < Kamal::Cli::Base
  desc "push", "Push the env file to the remote hosts"
  def push
-    mutating do
+    with_lock do
      on(KAMAL.hosts) do
        execute *KAMAL.auditor.record("Pushed env files"), verbosity: :debug

        KAMAL.roles_on(host).each do |role|
-          execute *KAMAL.app(role: role).make_env_directory
-          upload! role.env.secrets_io, role.env.secrets_file, mode: 400
+          execute *KAMAL.app(role: role, host: host).make_env_directory
+          upload! role.env(host).secrets_io, role.env(host).secrets_file, mode: 400
        end
      end

@@ -30,12 +30,12 @@ class Kamal::Cli::Env < Kamal::Cli::Base

  desc "delete", "Delete the env file from the remote hosts"
  def delete
-    mutating do
+    with_lock do
      on(KAMAL.hosts) do
        execute *KAMAL.auditor.record("Deleted env files"), verbosity: :debug

        KAMAL.roles_on(host).each do |role|
-          execute *KAMAL.app(role: role).remove_env_file
+          execute *KAMAL.app(role: role, host: host).remove_env_file
        end
      end

--- a/lib/kamal/cli/healthcheck.rb
+++ b/lib/kamal/cli/healthcheck.rb
@@ -1,21 +0,0 @@
-class Kamal::Cli::Healthcheck < Kamal::Cli::Base
-  default_command :perform
-
-  desc "perform", "Health check current app version"
-  def perform
-    raise "The primary host is not configured to run Traefik" unless KAMAL.config.role(KAMAL.config.primary_role).running_traefik?
-    on(KAMAL.primary_host) do
-      begin
-        execute *KAMAL.healthcheck.run
-        Poller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
-      rescue Poller::HealthcheckError => e
-        error capture_with_info(*KAMAL.healthcheck.logs)
-        error capture_with_pretty_json(*KAMAL.healthcheck.container_health_log)
-        raise
-      ensure
-        execute *KAMAL.healthcheck.stop, raise_on_non_zero_exit: false
-        execute *KAMAL.healthcheck.remove, raise_on_non_zero_exit: false
-      end
-    end
-  end
-end
--- a/lib/kamal/cli/healthcheck/barrier.rb
+++ b/lib/kamal/cli/healthcheck/barrier.rb
@@ -0,0 +1,31 @@
+class Kamal::Cli::Healthcheck::Barrier
+  def initialize
+    @ivar = Concurrent::IVar.new
+  end
+
+  def close
+    set(false)
+  end
+
+  def open
+    set(true)
+  end
+
+  def wait
+    unless opened?
+      raise Kamal::Cli::Healthcheck::Error.new("Halted at barrier")
+    end
+  end
+
+  private
+    def opened?
+      @ivar.value
+    end
+
+    def set(value)
+      @ivar.set(value)
+      true
+    rescue Concurrent::MultipleAssignmentError
+      false
+    end
+end
--- a/lib/kamal/cli/healthcheck/error.rb
+++ b/lib/kamal/cli/healthcheck/error.rb
@@ -0,0 +1,2 @@
+class Kamal::Cli::Healthcheck::Error < StandardError
+end
--- a/lib/kamal/cli/healthcheck/poller.rb
+++ b/lib/kamal/cli/healthcheck/poller.rb
@@ -3,7 +3,6 @@ module Kamal::Cli::Healthcheck::Poller

  TRAEFIK_UPDATE_DELAY = 5

-  class HealthcheckError < StandardError; end

  def wait_for_healthy(pause_after_ready: false, &block)
    attempt = 1
@@ -16,9 +15,9 @@ module Kamal::Cli::Healthcheck::Poller
      when "running" # No health check configured
        sleep KAMAL.config.readiness_delay if pause_after_ready
      else
-        raise HealthcheckError, "container not ready (#{status})"
+        raise Kamal::Cli::Healthcheck::Error, "container not ready (#{status})"
      end
-    rescue HealthcheckError => e
+    rescue Kamal::Cli::Healthcheck::Error => e
      if attempt <= max_attempts
        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
        sleep attempt
@@ -41,9 +40,9 @@ module Kamal::Cli::Healthcheck::Poller
      when "unhealthy"
        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
      else
-        raise HealthcheckError, "container not unhealthy (#{status})"
+        raise Kamal::Cli::Healthcheck::Error, "container not unhealthy (#{status})"
      end
-    rescue HealthcheckError => e
+    rescue Kamal::Cli::Healthcheck::Error => e
      if attempt <= max_attempts
        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
        sleep attempt
--- a/lib/kamal/cli/main.rb
+++ b/lib/kamal/cli/main.rb
@@ -3,14 +3,14 @@ class Kamal::Cli::Main < Kamal::Cli::Base
  option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
  def setup
    print_runtime do
-      mutating do
+      with_lock do
        invoke_options = deploy_options

        say "Ensure Docker is installed...", :magenta
        invoke "kamal:cli:server:bootstrap", [], invoke_options

-        say "Push env files...", :magenta
-        invoke "kamal:cli:env:push", [], invoke_options
+        say "Evaluate and push env files...", :magenta
+        invoke "kamal:cli:main:envify", [], invoke_options

        invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options
        deploy
@@ -22,30 +22,25 @@ class Kamal::Cli::Main < Kamal::Cli::Base
  option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
  def deploy
    runtime = print_runtime do
-      mutating do
-        invoke_options = deploy_options
+      invoke_options = deploy_options

-        say "Log into image registry...", :magenta
-        invoke "kamal:cli:registry:login", [], invoke_options
+      say "Log into image registry...", :magenta
+      invoke "kamal:cli:registry:login", [], invoke_options

-        if options[:skip_push]
-          say "Pull app image...", :magenta
-          invoke "kamal:cli:build:pull", [], invoke_options
-        else
-          say "Build and push app image...", :magenta
-          invoke "kamal:cli:build:deliver", [], invoke_options
-        end
+      if options[:skip_push]
+        say "Pull app image...", :magenta
+        invoke "kamal:cli:build:pull", [], invoke_options
+      else
+        say "Build and push app image...", :magenta
+        invoke "kamal:cli:build:deliver", [], invoke_options
+      end

+      with_lock do
        run_hook "pre-deploy"

        say "Ensure Traefik is running...", :magenta
        invoke "kamal:cli:traefik:boot", [], invoke_options

-        if KAMAL.config.role(KAMAL.config.primary_role).running_traefik?
-          say "Ensure app can pass healthcheck...", :magenta
-          invoke "kamal:cli:healthcheck:perform", [], invoke_options
-        end
-
        say "Detect stale containers...", :magenta
        invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)

@@ -63,22 +58,19 @@ class Kamal::Cli::Main < Kamal::Cli::Base
  option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
  def redeploy
    runtime = print_runtime do
-      mutating do
-        invoke_options = deploy_options
+      invoke_options = deploy_options

-        if options[:skip_push]
-          say "Pull app image...", :magenta
-          invoke "kamal:cli:build:pull", [], invoke_options
-        else
-          say "Build and push app image...", :magenta
-          invoke "kamal:cli:build:deliver", [], invoke_options
-        end
+      if options[:skip_push]
+        say "Pull app image...", :magenta
+        invoke "kamal:cli:build:pull", [], invoke_options
+      else
+        say "Build and push app image...", :magenta
+        invoke "kamal:cli:build:deliver", [], invoke_options
+      end

+      with_lock do
        run_hook "pre-deploy"

-        say "Ensure app can pass healthcheck...", :magenta
-        invoke "kamal:cli:healthcheck:perform", [], invoke_options
-
        say "Detect stale containers...", :magenta
        invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)

@@ -93,7 +85,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
  def rollback(version)
    rolled_back = false
    runtime = print_runtime do
-      mutating do
+      with_lock do
        invoke_options = deploy_options

        KAMAL.config.version = version
@@ -185,19 +177,23 @@ class Kamal::Cli::Main < Kamal::Cli::Base
      env_path          = ".env"
    end

-    File.write(env_path, ERB.new(File.read(env_template_path), trim_mode: "-").result, perm: 0600)
+    if Pathname.new(File.expand_path(env_template_path)).exist?
+      File.write(env_path, ERB.new(File.read(env_template_path), trim_mode: "-").result, perm: 0600)

-    unless options[:skip_push]
-      reload_envs
-      invoke "kamal:cli:env:push", options
+      unless options[:skip_push]
+        reload_envs
+        invoke "kamal:cli:env:push", options
+      end
+    else
+      puts "Skipping envify (no #{env_template_path} exist)"
    end
  end

  desc "remove", "Remove Traefik, app, accessories, and registry session from servers"
  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
  def remove
-    mutating do
-      confirming "This will remove all containers and images. Are you sure?" do
+    confirming "This will remove all containers and images. Are you sure?" do
+      with_lock do
        invoke "kamal:cli:traefik:remove", [], options.without(:confirmed)
        invoke "kamal:cli:app:remove", [], options.without(:confirmed)
        invoke "kamal:cli:accessory:remove", [ "all" ], options
@@ -223,9 +219,6 @@ class Kamal::Cli::Main < Kamal::Cli::Base
  desc "env", "Manage environment files"
  subcommand "env", Kamal::Cli::Env

-  desc "healthcheck", "Healthcheck application"
-  subcommand "healthcheck", Kamal::Cli::Healthcheck
-
  desc "lock", "Manage the deploy lock"
  subcommand "lock", Kamal::Cli::Lock

@@ -246,11 +239,11 @@ class Kamal::Cli::Main < Kamal::Cli::Base
      begin
        on(KAMAL.hosts) do
          KAMAL.roles_on(host).each do |role|
-            container_id = capture_with_info(*KAMAL.app(role: role).container_id_for_version(version))
+            container_id = capture_with_info(*KAMAL.app(role: role, host: host).container_id_for_version(version))
            raise "Container not found" unless container_id.present?
          end
        end
-      rescue SSHKit::Runner::ExecuteError => e
+      rescue SSHKit::Runner::ExecuteError, SSHKit::Runner::MultipleExecuteError => e
        if e.message =~ /Container not found/
          say "Error looking for container version #{version}: #{e.message}"
          return false
--- a/lib/kamal/cli/prune.rb
+++ b/lib/kamal/cli/prune.rb
@@ -1,7 +1,7 @@
 class Kamal::Cli::Prune < Kamal::Cli::Base
  desc "all", "Prune unused images and stopped containers"
  def all
-    mutating do
+    with_lock do
      containers
      images
    end
@@ -9,7 +9,7 @@ class Kamal::Cli::Prune < Kamal::Cli::Base

  desc "images", "Prune unused images"
  def images
-    mutating do
+    with_lock do
      on(KAMAL.hosts) do
        execute *KAMAL.auditor.record("Pruned images"), verbosity: :debug
        execute *KAMAL.prune.dangling_images
@@ -24,7 +24,7 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
    retain = options.fetch(:retain, KAMAL.config.retain_containers)
    raise "retain must be at least 1" if retain < 1

-    mutating do
+    with_lock do
      on(KAMAL.hosts) do
        execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug
        execute *KAMAL.prune.app_containers(retain: retain)
--- a/lib/kamal/cli/server.rb
+++ b/lib/kamal/cli/server.rb
@@ -1,25 +1,49 @@
 class Kamal::Cli::Server < Kamal::Cli::Base
+  desc "exec", "Run a custom command on the server (use --help to show options)"
+  option :interactive, type: :boolean, aliases: "-i", default: false, desc: "Run the command interactively (use for console/bash)"
+  def exec(cmd)
+    hosts = KAMAL.hosts | KAMAL.accessory_hosts
+
+    case
+    when options[:interactive]
+      host = KAMAL.primary_host
+
+      say "Running '#{cmd}' on #{host} interactively...", :magenta
+
+      run_locally { exec KAMAL.server.run_over_ssh(cmd, host: host) }
+    else
+      say "Running '#{cmd}' on #{hosts.join(', ')}...", :magenta
+
+      on(hosts) do |host|
+        execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{host}"), verbosity: :debug
+        puts_by_host host, capture_with_info(cmd)
+      end
+    end
+  end
+
  desc "bootstrap", "Set up Docker to run Kamal apps"
  def bootstrap
-    missing = []
+    with_lock do
+      missing = []

-    on(KAMAL.hosts | KAMAL.accessory_hosts) do |host|
-      unless execute(*KAMAL.docker.installed?, raise_on_non_zero_exit: false)
-        if execute(*KAMAL.docker.superuser?, raise_on_non_zero_exit: false)
-          info "Missing Docker on #{host}. Installing…"
-          execute *KAMAL.docker.install
-        else
-          missing << host
+      on(KAMAL.hosts | KAMAL.accessory_hosts) do |host|
+        unless execute(*KAMAL.docker.installed?, raise_on_non_zero_exit: false)
+          if execute(*KAMAL.docker.superuser?, raise_on_non_zero_exit: false)
+            info "Missing Docker on #{host}. Installing…"
+            execute *KAMAL.docker.install
+          else
+            missing << host
+          end
        end
+
+        execute(*KAMAL.server.ensure_run_directory)
      end

-      execute(*KAMAL.server.ensure_run_directory)
-    end
+      if missing.any?
+        raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and either `wget` or `curl`. Install Docker manually: https://docs.docker.com/engine/install/"
+      end

-    if missing.any?
-      raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and either `wget` or `curl`. Install Docker manually: https://docs.docker.com/engine/install/"
+      run_hook "docker-setup"
    end
-
-    run_hook "docker-setup"
  end
 end
--- a/lib/kamal/cli/traefik.rb
+++ b/lib/kamal/cli/traefik.rb
@@ -1,7 +1,7 @@
 class Kamal::Cli::Traefik < Kamal::Cli::Base
  desc "boot", "Boot Traefik on servers"
  def boot
-    mutating do
+    with_lock do
      on(KAMAL.traefik_hosts) do
        execute *KAMAL.registry.login
        execute *KAMAL.traefik.start_or_run
@@ -14,7 +14,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
  def reboot
    confirming "This will cause a brief outage on each host. Are you sure?" do
-      mutating do
+      with_lock do
        host_groups = options[:rolling] ? KAMAL.traefik_hosts : [ KAMAL.traefik_hosts ]
        host_groups.each do |hosts|
          host_list = Array(hosts).join(",")
@@ -34,7 +34,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base

  desc "start", "Start existing Traefik container on servers"
  def start
-    mutating do
+    with_lock do
      on(KAMAL.traefik_hosts) do
        execute *KAMAL.auditor.record("Started traefik"), verbosity: :debug
        execute *KAMAL.traefik.start
@@ -44,7 +44,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base

  desc "stop", "Stop existing Traefik container on servers"
  def stop
-    mutating do
+    with_lock do
      on(KAMAL.traefik_hosts) do
        execute *KAMAL.auditor.record("Stopped traefik"), verbosity: :debug
        execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
@@ -54,7 +54,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base

  desc "restart", "Restart existing Traefik container on servers"
  def restart
-    mutating do
+    with_lock do
      stop
      start
    end
@@ -91,7 +91,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base

  desc "remove", "Remove Traefik container and image from servers"
  def remove
-    mutating do
+    with_lock do
      stop
      remove_container
      remove_image
@@ -100,7 +100,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base

  desc "remove_container", "Remove Traefik container from servers", hide: true
  def remove_container
-    mutating do
+    with_lock do
      on(KAMAL.traefik_hosts) do
        execute *KAMAL.auditor.record("Removed traefik container"), verbosity: :debug
        execute *KAMAL.traefik.remove_container
@@ -110,7 +110,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base

  desc "remove_image", "Remove Traefik image from servers", hide: true
  def remove_image
-    mutating do
+    with_lock do
      on(KAMAL.traefik_hosts) do
        execute *KAMAL.auditor.record("Removed traefik image"), verbosity: :debug
        execute *KAMAL.traefik.remove_image
--- a/lib/kamal/commander.rb
+++ b/lib/kamal/commander.rb
@@ -2,13 +2,13 @@ require "active_support/core_ext/enumerable"
 require "active_support/core_ext/module/delegation"

 class Kamal::Commander
-  attr_accessor :verbosity, :holding_lock, :hold_lock_on_error
+  attr_accessor :verbosity, :holding_lock, :connected
  delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :traefik_hosts, :accessory_hosts, to: :specifics

  def initialize
    self.verbosity = :info
    self.holding_lock = false
-    self.hold_lock_on_error = false
+    self.connected = false
    @specifics = nil
  end

@@ -65,8 +65,8 @@ class Kamal::Commander
  end


-  def app(role: nil)
-    Kamal::Commands::App.new(config, role: role)
+  def app(role: nil, host: nil)
+    Kamal::Commands::App.new(config, role: role, host: host)
  end

  def accessory(name)
@@ -138,8 +138,8 @@ class Kamal::Commander
    self.holding_lock
  end

-  def hold_lock_on_error?
-    self.hold_lock_on_error
+  def connected?
+    self.connected
  end

  private
--- a/lib/kamal/commands/app.rb
+++ b/lib/kamal/commands/app.rb
@@ -3,11 +3,12 @@ class Kamal::Commands::App < Kamal::Commands::Base

  ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]

-  attr_reader :role, :role
+  attr_reader :role, :host

-  def initialize(config, role: nil)
+  def initialize(config, role: nil, host: nil)
    super(config)
    @role = role
+    @host = host
  end

  def run(hostname: nil)
@@ -18,7 +19,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
      *([ "--hostname", hostname ] if hostname),
      "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
      "-e", "KAMAL_VERSION=\"#{config.version}\"",
-      *role.env_args,
+      *role.env_args(host),
      *role.health_check_args,
      *role.logging_args,
      *config.volume_args,
@@ -70,11 +71,11 @@ class Kamal::Commands::App < Kamal::Commands::Base


  def make_env_directory
-    make_directory role.env.secrets_directory
+    make_directory role.env(host).secrets_directory
  end

  def remove_env_file
-    [ :rm, "-f", role.env.secrets_file ]
+    [ :rm, "-f", role.env(host).secrets_file ]
  end


--- a/lib/kamal/commands/app/containers.rb
+++ b/lib/kamal/commands/app/containers.rb
@@ -1,4 +1,6 @@
 module Kamal::Commands::App::Containers
+  DOCKER_HEALTH_LOG_FORMAT    = "'{{json .State.Health}}'"
+
  def list_containers
    docker :container, :ls, "--all", *filter_args
  end
@@ -20,4 +22,10 @@ module Kamal::Commands::App::Containers
  def remove_containers
    docker :container, :prune, "--force", *filter_args
  end
+
+  def container_health_log(version:)
+    pipe \
+      container_id_for(container_name: container_name(version)),
+      xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT))
+  end
 end
--- a/lib/kamal/commands/app/execution.rb
+++ b/lib/kamal/commands/app/execution.rb
@@ -11,7 +11,7 @@ module Kamal::Commands::App::Execution
    docker :run,
      ("-it" if interactive),
      "--rm",
-      *role&.env_args,
+      *role&.env_args(host),
      *argumentize("--env", env),
      *config.volume_args,
      *role&.option_args,
@@ -19,11 +19,11 @@ module Kamal::Commands::App::Execution
      *command
  end

-  def execute_in_existing_container_over_ssh(*command, host:, env:)
+  def execute_in_existing_container_over_ssh(*command,  env:)
    run_over_ssh execute_in_existing_container(*command, interactive: true, env: env), host: host
  end

-  def execute_in_new_container_over_ssh(*command, host:, env:)
+  def execute_in_new_container_over_ssh(*command, env:)
    run_over_ssh execute_in_new_container(*command, interactive: true, env: env), host: host
  end
 end
--- a/lib/kamal/commands/app/logging.rb
+++ b/lib/kamal/commands/app/logging.rb
@@ -1,7 +1,7 @@
 module Kamal::Commands::App::Logging
-  def logs(since: nil, lines: nil, grep: nil)
+  def logs(version: nil, since: nil, lines: nil, grep: nil)
    pipe \
-      current_running_container_id,
+      version ? container_id_for_version(version) : current_running_container_id,
      "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
      ("grep '#{grep}'" if grep)
  end
--- a/lib/kamal/commands/base.rb
+++ b/lib/kamal/commands/base.rb
@@ -3,7 +3,6 @@ module Kamal::Commands
    delegate :sensitive, :argumentize, to: Kamal::Utils

    DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'"
-    DOCKER_HEALTH_LOG_FORMAT    = "'{{json .State.Health}}'"

    attr_accessor :config

@@ -78,8 +77,8 @@ module Kamal::Commands
        args.compact.unshift :docker
      end

-      def git(*args)
-        args.compact.unshift :git
+      def git(*args, path: nil)
+        [ :git, *([ "-C", path ] if path), *args.compact ]
      end

      def tags(**details)
--- a/lib/kamal/commands/builder.rb
+++ b/lib/kamal/commands/builder.rb
@@ -3,6 +3,8 @@ require "active_support/core_ext/string/filters"
 class Kamal::Commands::Builder < Kamal::Commands::Base
  delegate :create, :remove, :push, :clean, :pull, :info, :validate_image, to: :target

+  include Clone
+
  def name
    target.class.to_s.remove("Kamal::Commands::Builder::").underscore.inquiry
  end
--- a/lib/kamal/commands/builder/base.rb
+++ b/lib/kamal/commands/builder/base.rb
@@ -3,7 +3,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
  class BuilderError < StandardError; end

  delegate :argumentize, to: Kamal::Utils
-  delegate :args, :secrets, :dockerfile, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, :ssh, :git_archive?, to: :builder_config
+  delegate :args, :secrets, :dockerfile, :target, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, :ssh, to: :builder_config

  def clean
    docker :image, :rm, "--force", config.absolute_image
@@ -13,18 +13,8 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
    docker :pull, config.absolute_image
  end

-  def push
-    if git_archive?
-      pipe \
-        git(:archive, "--format=tar", :HEAD),
-        build_and_push
-    else
-      build_and_push
-    end
-  end
-
  def build_options
-    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_ssh ]
+    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_target, *build_ssh ]
  end

  def build_context
@@ -73,6 +63,10 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
      end
    end

+    def build_target
+      argumentize "--target", target if target.present?
+    end
+
    def build_ssh
      argumentize "--ssh", ssh if ssh.present?
    end
--- a/lib/kamal/commands/builder/clone.rb
+++ b/lib/kamal/commands/builder/clone.rb
@@ -0,0 +1,28 @@
+module Kamal::Commands::Builder::Clone
+  extend ActiveSupport::Concern
+
+  included do
+    delegate :clone_directory, :build_directory, to: :"config.builder"
+  end
+
+  def clone
+    git :clone, Kamal::Git.root, path: clone_directory
+  end
+
+  def clone_reset_steps
+    [
+      git(:remote, "set-url", :origin, Kamal::Git.root, path: build_directory),
+      git(:fetch, :origin, path: build_directory),
+      git(:reset, "--hard", Kamal::Git.revision, path: build_directory),
+      git(:clean, "-fdx", path: build_directory)
+    ]
+  end
+
+  def clone_status
+    git :status, "--porcelain", path: build_directory
+  end
+
+  def clone_revision
+    git :"rev-parse", :HEAD, path: build_directory
+  end
+end
--- a/lib/kamal/commands/builder/multiarch.rb
+++ b/lib/kamal/commands/builder/multiarch.rb
@@ -13,6 +13,15 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
      docker(:buildx, :ls)
  end

+  def push
+    docker :buildx, :build,
+      "--push",
+      "--platform", platform_names,
+      "--builder", builder_name,
+      *build_options,
+      build_context
+  end
+
  private
    def builder_name
      "kamal-#{config.service}-multiarch"
@@ -25,13 +34,4 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
        "linux/amd64,linux/arm64"
      end
    end
-
-    def build_and_push
-      docker :buildx, :build,
-        "--push",
-        "--platform", platform_names,
-        "--builder", builder_name,
-        *build_options,
-        build_context
-    end
 end
--- a/lib/kamal/commands/builder/native.rb
+++ b/lib/kamal/commands/builder/native.rb
@@ -11,11 +11,10 @@ class Kamal::Commands::Builder::Native < Kamal::Commands::Builder::Base
    # No-op on native
  end

-  private
-    def build_and_push
-      combine \
-        docker(:build, *build_options, build_context),
-        docker(:push, config.absolute_image),
-        docker(:push, config.latest_image)
-    end
+  def push
+    combine \
+      docker(:build, *build_options, build_context),
+      docker(:push, config.absolute_image),
+      docker(:push, config.latest_image)
+  end
 end
--- a/lib/kamal/commands/builder/native/cached.rb
+++ b/lib/kamal/commands/builder/native/cached.rb
@@ -7,11 +7,10 @@ class Kamal::Commands::Builder::Native::Cached < Kamal::Commands::Builder::Nativ
    docker :buildx, :rm, builder_name
  end

-  private
-    def build_and_push
-      docker :buildx, :build,
-        "--push",
-        *build_options,
-        build_context
-    end
+  def push
+    docker :buildx, :build,
+      "--push",
+      *build_options,
+      build_context
+  end
 end
--- a/lib/kamal/commands/builder/native/remote.rb
+++ b/lib/kamal/commands/builder/native/remote.rb
@@ -17,6 +17,15 @@ class Kamal::Commands::Builder::Native::Remote < Kamal::Commands::Builder::Nativ
      docker(:buildx, :ls)
  end

+  def push
+    docker :buildx, :build,
+    "--push",
+    "--platform", platform,
+    "--builder", builder_name,
+    *build_options,
+    build_context
+  end
+

  private
    def builder_name
@@ -47,13 +56,4 @@ class Kamal::Commands::Builder::Native::Remote < Kamal::Commands::Builder::Nativ
    def remove_buildx
      docker :buildx, :rm, builder_name
    end
-
-    def build_and_push
-      docker :buildx, :build,
-      "--push",
-      "--platform", platform,
-      "--builder", builder_name,
-      *build_options,
-      build_context
-    end
 end
--- a/lib/kamal/commands/healthcheck.rb
+++ b/lib/kamal/commands/healthcheck.rb
@@ -1,59 +0,0 @@
-class Kamal::Commands::Healthcheck < Kamal::Commands::Base
-  def run
-    primary = config.role(config.primary_role)
-
-    docker :run,
-      "--detach",
-      "--name", container_name_with_version,
-      "--publish", "#{exposed_port}:#{config.healthcheck["port"]}",
-      "--label", "service=#{config.healthcheck_service}",
-      "-e", "KAMAL_CONTAINER_NAME=\"#{config.healthcheck_service}\"",
-      *primary.env_args,
-      *primary.health_check_args(cord: false),
-      *config.volume_args,
-      *primary.option_args,
-      config.absolute_image,
-      primary.cmd
-  end
-
-  def status
-    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_STATUS_FORMAT))
-  end
-
-  def container_health_log
-    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT))
-  end
-
-  def logs
-    pipe container_id, xargs(docker(:logs, "--tail", log_lines, "2>&1"))
-  end
-
-  def stop
-    pipe container_id, xargs(docker(:stop))
-  end
-
-  def remove
-    pipe container_id, xargs(docker(:container, :rm))
-  end
-
-  private
-    def container_name_with_version
-      "#{config.healthcheck_service}-#{config.version}"
-    end
-
-    def container_id
-      container_id_for(container_name: container_name_with_version)
-    end
-
-    def health_url
-      "http://localhost:#{exposed_port}#{config.healthcheck["path"]}"
-    end
-
-    def exposed_port
-      config.healthcheck["exposed_port"]
-    end
-
-    def log_lines
-      config.healthcheck["log_lines"]
-    end
-end
--- a/lib/kamal/configuration.rb
+++ b/lib/kamal/configuration.rb
@@ -188,7 +188,7 @@ class Kamal::Configuration


  def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }.merge(raw_config.healthcheck || {})
+    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }.merge(raw_config.healthcheck || {})
  end

  def healthcheck_service
@@ -233,6 +233,18 @@ class Kamal::Configuration
    raw_config.env || {}
  end

+  def env_tags
+    @env_tags ||= if (tags = raw_config.env["tags"])
+      tags.collect { |name, config| Kamal::Configuration::Env::Tag.new(name, config: config) }
+    else
+      []
+    end
+  end
+
+  def env_tag(name)
+    env_tags.detect { |t| t.name == name.to_s }
+  end
+

  def valid?
    ensure_destination_if_required && ensure_required_keys_present && ensure_valid_kamal_version && ensure_retain_containers_valid && ensure_valid_service_name
@@ -328,7 +340,7 @@ class Kamal::Configuration
    def git_version
      @git_version ||=
        if Kamal::Git.used?
-          if Kamal::Git.uncommitted_changes.present? && !builder.git_archive?
+          if Kamal::Git.uncommitted_changes.present? && !builder.git_clone?
            uncommitted_suffix = "_uncommitted_#{SecureRandom.hex(8)}"
          end
          [ Kamal::Git.revision, uncommitted_suffix ].compact.join
--- a/lib/kamal/configuration/builder.rb
+++ b/lib/kamal/configuration/builder.rb
@@ -3,6 +3,8 @@ class Kamal::Configuration::Builder
    @options = config.raw_config.builder || {}
    @image = config.image
    @server = config.registry["server"]
+    @service = config.service
+    @destination = config.destination

    valid?
  end
@@ -39,8 +41,12 @@ class Kamal::Configuration::Builder
    @options["dockerfile"] || "Dockerfile"
  end

+  def target
+    @options["target"]
+  end
+
  def context
-    @options["context"] || (git_archive? ? "-" : ".")
+    @options["context"] || "."
  end

  def local_arch
@@ -85,10 +91,23 @@ class Kamal::Configuration::Builder
    @options["ssh"]
  end

-  def git_archive?
+  def git_clone?
    Kamal::Git.used? && @options["context"].nil?
  end

+  def clone_directory
+    @clone_directory ||= File.join Dir.tmpdir, "kamal-clones", [ @service, pwd_sha ].compact.join("-")
+  end
+
+  def build_directory
+    @build_directory ||=
+      if git_clone?
+        File.join clone_directory, repo_basename, repo_relative_pwd
+      else
+        "."
+      end
+  end
+
  private
    def valid?
      if @options["cache"] && @options["cache"]["type"]
@@ -119,4 +138,16 @@ class Kamal::Configuration::Builder
    def cache_to_config_for_registry
      [ "type=registry", @options["cache"]&.fetch("options", nil), "ref=#{cache_image_ref}" ].compact.join(",")
    end
+
+    def repo_basename
+      File.basename(Kamal::Git.root)
+    end
+
+    def repo_relative_pwd
+      Dir.pwd.delete_prefix(Kamal::Git.root)
+    end
+
+    def pwd_sha
+      Digest::SHA256.hexdigest(Dir.pwd)[0..12]
+    end
 end
--- a/lib/kamal/configuration/env.rb
+++ b/lib/kamal/configuration/env.rb
@@ -4,7 +4,7 @@ class Kamal::Configuration::Env

  def self.from_config(config:, secrets_file: nil)
    secrets_keys = config.fetch("secret", [])
-    clear = config.fetch("clear", config.key?("secret") ? {} : config)
+    clear = config.fetch("clear", config.key?("secret") || config.key?("tags") ? {} : config)

    new clear: clear, secrets_keys: secrets_keys, secrets_file: secrets_file
  end
--- a/lib/kamal/configuration/env/tag.rb
+++ b/lib/kamal/configuration/env/tag.rb
@@ -0,0 +1,12 @@
+class Kamal::Configuration::Env::Tag
+  attr_reader :name, :config
+
+  def initialize(name, config:)
+    @name = name
+    @config = config
+  end
+
+  def env
+    Kamal::Configuration::Env.from_config(config: config)
+  end
+end
--- a/lib/kamal/configuration/role.rb
+++ b/lib/kamal/configuration/role.rb
@@ -7,6 +7,7 @@ class Kamal::Configuration::Role

  def initialize(name, config:)
    @name, @config = name.inquiry, config
+    @tagged_hosts ||= extract_tagged_hosts_from_config
  end

  def primary_host
@@ -14,7 +15,11 @@ class Kamal::Configuration::Role
  end

  def hosts
-    @hosts ||= extract_hosts_from_config
+    tagged_hosts.keys
+  end
+
+  def env_tags(host)
+    tagged_hosts.fetch(host).collect { |tag| config.env_tag(tag) }
  end

  def cmd
@@ -50,12 +55,13 @@ class Kamal::Configuration::Role
  end


-  def env
-    @env ||= base_env.merge(specialized_env)
+  def env(host)
+    @envs ||= {}
+    @envs[host] ||= [ base_env, specialized_env, *env_tags(host).map(&:env) ].reduce(:merge)
  end

-  def env_args
-    env.args
+  def env_args(host)
+    env(host).args
  end

  def asset_volume_args
@@ -164,7 +170,24 @@ class Kamal::Configuration::Role
  end

  private
-    attr_accessor :config
+    attr_accessor :config, :tagged_hosts
+
+    def extract_tagged_hosts_from_config
+      {}.tap do |tagged_hosts|
+        extract_hosts_from_config.map do |host_config|
+          if host_config.is_a?(Hash)
+            raise ArgumentError, "Multiple hosts found: #{host_config.inspect}" unless host_config.size == 1
+
+            host, tags = host_config.first
+            tagged_hosts[host] = Array(tags)
+          elsif host_config.is_a?(String) || host_config.is_a?(Symbol)
+            tagged_hosts[host_config] = []
+          else
+            raise ArgumentError, "Invalid host config: #{host_config.inspect}"
+          end
+        end
+      end
+    end

    def extract_hosts_from_config
      if config.servers.is_a?(Array)
--- a/lib/kamal/git.rb
+++ b/lib/kamal/git.rb
@@ -16,4 +16,8 @@ module Kamal::Git
  def uncommitted_changes
    `git status --porcelain`.strip
  end
+
+  def root
+    `git rev-parse --show-toplevel`.strip
+  end
 end
--- a/lib/kamal/sshkit_with_ext.rb
+++ b/lib/kamal/sshkit_with_ext.rb
@@ -103,3 +103,39 @@ class SSHKit::Backend::Netssh

  prepend LimitConcurrentStartsInstance
 end
+
+class SSHKit::Runner::Parallel
+  # SSHKit joins the threads in sequence and fails on the first error it encounters, which means that we wait threads
+  # before the first failure to complete but not for ones after.
+  #
+  # We'll patch it to wait for them all to complete, and to record all the threads that errored so we can see when a
+  # problem occurs on multiple hosts.
+  module CompleteAll
+    def execute
+      threads = hosts.map do |host|
+        Thread.new(host) do |h|
+          backend(h, &block).run
+        rescue ::StandardError => e
+          e2 = SSHKit::Runner::ExecuteError.new e
+          raise e2, "Exception while executing #{host.user ? "as #{host.user}@" : "on host "}#{host}: #{e.message}"
+        end
+      end
+
+      exceptions = []
+      threads.each do |t|
+        begin
+          t.join
+        rescue SSHKit::Runner::ExecuteError => e
+          exceptions << e
+        end
+      end
+      if exceptions.one?
+        raise exceptions.first
+      elsif exceptions.many?
+        raise exceptions.first, [ "Exceptions on #{exceptions.count} hosts:", exceptions.map(&:message) ].join("\n")
+      end
+    end
+  end
+
+  prepend CompleteAll
+end
--- a/lib/kamal/version.rb
+++ b/lib/kamal/version.rb
@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "1.5.2"
+  VERSION = "1.6.0"
 end
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -54,14 +54,14 @@ class CliAppTest < CliTestCase
    run_command("boot", config: :with_boot_strategy)
  end

-  test "boot errors leave lock in place" do
+  test "boot errors don't leave lock in place" do
    Kamal::Cli::App.any_instance.expects(:using_version).raises(RuntimeError)

    assert_not KAMAL.holding_lock?
    assert_raises(RuntimeError) do
      stderred { run_command("boot") }
    end
-    assert KAMAL.holding_lock?
+    assert_not KAMAL.holding_lock?
  end

  test "boot with assets" do
@@ -92,6 +92,82 @@ class CliAppTest < CliTestCase
    end
  end

+  test "boot with host tags" do
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+    .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+    .returns("12345678") # running version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
+      .returns("") # old version
+
+    run_command("boot", config: :with_env_tags).tap do |output|
+      assert_match "docker tag dhh/app:latest dhh/app:latest", output
+      assert_match %r{docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} -e KAMAL_CONTAINER_NAME="app-web-latest" -e KAMAL_VERSION="latest" --env-file .kamal/env/roles/app-web.env --env TEST="root" --env EXPERIMENT="disabled" --env SITE="site1"}, output
+      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+    end
+  end
+
+  test "boot with web barrier opened" do
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running").at_least_once # web health check passing
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("unhealthy").at_least_once # web health check failing
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running").at_least_once # workers health check
+
+    run_command("boot", config: :with_roles, host: nil).tap do |output|
+      assert_match "Waiting for the first healthy web container before booting workers on 1.1.1.3...", output
+      assert_match "Waiting for the first healthy web container before booting workers on 1.1.1.4...", output
+      assert_match "First web container is healthy, booting workers on 1.1.1.3", output
+      assert_match "First web container is healthy, booting workers on 1.1.1.4", output
+  end
+  end
+
+  test "boot with web barrier closed" do
+    Thread.report_on_exception = false
+
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("unhealthy").at_least_once # web health check failing
+
+    stderred do
+      run_command("boot", config: :with_roles, host: nil, allow_execute_error: true).tap do |output|
+        assert_match "Waiting for the first healthy web container before booting workers on 1.1.1.3...", output
+        assert_match "Waiting for the first healthy web container before booting workers on 1.1.1.4...", output
+        assert_match "First web container is unhealthy, not booting workers on 1.1.1.3", output
+        assert_match "First web container is unhealthy, not booting workers on 1.1.1.4", output
+        assert_match "Running docker container ls --all --filter name=^app-web-latest$ --quiet | xargs docker stop on 1.1.1.1", output
+        assert_match "Running docker container ls --all --filter name=^app-web-latest$ --quiet | xargs docker stop on 1.1.1.2", output
+      end
+    end
+  ensure
+    Thread.report_on_exception = true
+  end
+
  test "start" do
    run_command("start").tap do |output|
      assert_match "docker start app-web-999", output
@@ -236,9 +312,33 @@ class CliAppTest < CliTestCase
    end
  end

+  test "long hostname" do
+    stub_running
+
+    hostname = "this-hostname-is-really-unacceptably-long-to-be-honest.example.com"
+
+    stdouted { Kamal::Cli::App.start([ "boot", "-c", "test/fixtures/deploy_with_uncommon_hostnames.yml", "--hosts", hostname ]) }.tap do |output|
+      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname this-hostname-is-really-unacceptably-long-to-be-hon-[0-9a-f]{12} /, output
+    end
+  end
+
+  test "hostname is trimmed if will end with a period" do
+    stub_running
+
+    hostname = "this-hostname-with-random-part-is-too-long.example.com"
+
+    stdouted { Kamal::Cli::App.start([ "boot", "-c", "test/fixtures/deploy_with_uncommon_hostnames.yml", "--hosts", hostname ]) }.tap do |output|
+      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname this-hostname-with-random-part-is-too-long.example-[0-9a-f]{12} /, output
+    end
+  end
+
  private
-    def run_command(*command, config: :with_accessories)
-      stdouted { Kamal::Cli::App.start([ *command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1" ]) }
+    def run_command(*command, config: :with_accessories, host: "1.1.1.1", allow_execute_error: false)
+      stdouted do
+        Kamal::Cli::App.start([ *command, "-c", "test/fixtures/deploy_#{config}.yml", *([ "--hosts", host ] if host) ])
+      rescue SSHKit::Runner::ExecuteError => e
+        raise e unless allow_execute_error
+      end
    end

    def stub_running
--- a/test/cli/build_test.rb
+++ b/test/cli/build_test.rb
@@ -9,32 +9,137 @@ class CliBuildTest < CliTestCase
  end

  test "push" do
+    with_build_directory do |build_directory|
+      Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+      hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" }
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :"rev-parse", :HEAD)
+        .returns(Kamal::Git.revision)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :status, "--porcelain")
+        .returns("")
+
+      run_command("push", "--verbose").tap do |output|
+        assert_hook_ran "pre-build", output, **hook_variables
+        assert_match /Cloning repo into build directory/, output
+        assert_match /git -C #{Dir.tmpdir}\/kamal-clones\/app-#{pwd_sha} clone #{Dir.pwd}/, output
+        assert_match /docker --version && docker buildx version/, output
+        assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
+      end
+    end
+  end
+
+  test "push resetting clone" do
+    with_build_directory do |build_directory|
+      stub_setup
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd)
+        .raises(SSHKit::Command::Failed.new("fatal: destination path 'kamal' already exists and is not an empty directory"))
+        .then
+        .returns(true)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :remote, "set-url", :origin, Dir.pwd)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :fetch, :origin)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :reset, "--hard", Kamal::Git.revision)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :clean, "-fdx")
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with(:docker, :buildx, :build, "--push", "--platform", "linux/amd64,linux/arm64", "--builder", "kamal-app-multiarch", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".")
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :"rev-parse", :HEAD)
+        .returns(Kamal::Git.revision)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :status, "--porcelain")
+        .returns("")
+
+      run_command("push", "--verbose").tap do |output|
+        assert_match /Cloning repo into build directory/, output
+        assert_match /Resetting local clone/, output
+      end
+    end
+  end
+
+  test "push without clone" do
    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
    hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" }

-    run_command("push", "--verbose").tap do |output|
+    run_command("push", "--verbose", fixture: :without_clone).tap do |output|
+      assert_no_match /Cloning repo into build directory/, output
      assert_hook_ran "pre-build", output, **hook_variables
      assert_match /docker --version && docker buildx version/, output
-      assert_match /git archive -tar HEAD | docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile - as .*@localhost/, output
+      assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile . as .*@localhost/, output
+    end
+  end
+
+  test "push with corrupt clone" do
+    with_build_directory do |build_directory|
+      stub_setup
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd)
+        .raises(SSHKit::Command::Failed.new("fatal: destination path 'kamal' already exists and is not an empty directory"))
+        .then
+        .returns(true)
+        .twice
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :remote, "set-url", :origin, Dir.pwd)
+        .raises(SSHKit::Command::Failed.new("fatal: not a git repository"))
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :"rev-parse", :HEAD)
+        .returns(Kamal::Git.revision)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :status, "--porcelain")
+        .returns("")
+
+      Dir.stubs(:chdir)
+
+      run_command("push", "--verbose") do |output|
+        assert_match /Cloning repo into build directory `#{build_directory}`\.\.\..*Cloning repo into build directory `#{build_directory}`\.\.\./, output
+        assert_match "Resetting local clone as `#{build_directory}` already exists...", output
+        assert_match "Error preparing clone: Failed to clone repo: fatal: not a git repository, deleting and retrying...", output
+      end
    end
  end

  test "push without builder" do
-    stub_setup
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, "--version", "&&", :docker, :buildx, "version")
+    with_build_directory do |build_directory|
+      stub_setup

-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :buildx, :create, "--use", "--name", "kamal-app-multiarch")
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with(:docker, "--version", "&&", :docker, :buildx, "version")

-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*args| p args[0..6]; args[0..6] == [ :git, :archive, "--format=tar", :HEAD, "|", :docker, :buildx ] }
-      .raises(SSHKit::Command::Failed.new("no builder"))
-      .then
-      .returns(true)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with(:docker, :buildx, :create, "--use", "--name", "kamal-app-multiarch")

-    run_command("push").tap do |output|
-      assert_match /WARN Missing compatible builder, so creating a new one first/, output
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with { |*args| args[0..1] == [ :docker, :buildx ] }
+        .raises(SSHKit::Command::Failed.new("no builder"))
+        .then
+        .returns(true)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with { |*args| args.first.start_with?("git") }
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :"rev-parse", :HEAD)
+        .returns(Kamal::Git.revision)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :status, "--porcelain")
+        .returns("")
+
+      run_command("push").tap do |output|
+        assert_match /WARN Missing compatible builder, so creating a new one first/, output
+      end
    end
  end

@@ -79,6 +184,14 @@ class CliBuildTest < CliTestCase
    end
  end

+  test "create remote with custom ports" do
+    run_command("create", fixture: :with_remote_builder_and_custom_ports).tap do |output|
+      assert_match "Running /usr/bin/env true on 1.1.1.5", output
+      assert_match "docker context create kamal-app-native-remote-amd64 --description 'kamal-app-native-remote amd64 native host' --docker 'host=ssh://app@1.1.1.5:2122'", output
+      assert_match "docker buildx create --name kamal-app-native-remote kamal-app-native-remote-amd64 --platform linux/amd64", output
+    end
+  end
+
  test "create with error" do
    stub_setup
    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
@@ -118,4 +231,17 @@ class CliBuildTest < CliTestCase
      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
        .with { |*args| args[0..1] == [ :docker, :buildx ] }
    end
+
+    def with_build_directory
+      build_directory = File.join Dir.tmpdir, "kamal-clones", "app-#{pwd_sha}", "kamal"
+      FileUtils.mkdir_p build_directory
+      FileUtils.touch File.join build_directory, "Dockerfile"
+      yield build_directory + "/"
+    ensure
+      FileUtils.rm_rf build_directory
+    end
+
+    def pwd_sha
+      Digest::SHA256.hexdigest(Dir.pwd)[0..12]
+    end
 end
--- a/test/cli/healthcheck_test.rb
+++ b/test/cli/healthcheck_test.rb
@@ -1,82 +0,0 @@
-require_relative "cli_test_case"
-
-class CliHealthcheckTest < CliTestCase
-  test "perform" do
-    # Prevent expected failures from outputting to terminal
-    Thread.report_on_exception = false
-
-    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
-    Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
-
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--env-file", ".kamal/env/roles/app-web.env", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false)
-
-    # Fail twice to test retry logic
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-      .returns("starting")
-      .then
-      .returns("unhealthy")
-      .then
-      .returns("healthy")
-
-    run_command("perform").tap do |output|
-      assert_match "container not ready (starting), retrying in 1s (attempt 1/7)...", output
-      assert_match "container not ready (unhealthy), retrying in 2s (attempt 2/7)...", output
-      assert_match "Container is healthy!", output
-    end
-  end
-
-  test "perform failing to become healthy" do
-    # Prevent expected failures from outputting to terminal
-    Thread.report_on_exception = false
-
-    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
-
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--env-file", ".kamal/env/roles/app-web.env", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false)
-
-    # Continually report unhealthy
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-      .returns("unhealthy")
-
-    # Capture logs when failing
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :logs, "--tail", 50, "2>&1")
-      .returns("some log output")
-
-    # Capture container health log when failing
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_pretty_json)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :inspect, "--format",  "'{{json .State.Health}}'")
-      .returns('{"Status":"unhealthy","Log":[{"ExitCode": 1,"Output": "/bin/sh: 1: curl: not found\n"}]}"')
-
-    exception = assert_raises do
-      run_command("perform")
-    end
-    assert_match "container not ready (unhealthy)", exception.message
-  end
-
-  test "raises an exception if primary does not have traefik" do
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).never
-
-    exception = assert_raises do
-      run_command("perform", config_file: "test/fixtures/deploy_workers_only.yml")
-    end
-
-    assert_equal "The primary host is not configured to run Traefik", exception.message
-  end
-
-  private
-    def run_command(*command, config_file: "test/fixtures/deploy_with_accessories.yml")
-      stdouted { Kamal::Cli::Healthcheck.start([ *command, "-c", config_file ]) }
-    end
-end
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -5,13 +5,13 @@ class CliMainTest < CliTestCase
    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }

    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:main:envify", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:deploy)

    run_command("setup").tap do |output|
      assert_match /Ensure Docker is installed.../, output
-      assert_match /Push env files.../, output
+      assert_match /Evaluate and push env files.../, output
    end
  end

@@ -19,26 +19,24 @@ class CliMainTest < CliTestCase
    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }

    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:main:envify", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
    # deploy
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)

    run_command("setup", "--skip_push").tap do |output|
      assert_match /Ensure Docker is installed.../, output
-      assert_match /Push env files.../, output
+      assert_match /Evaluate and push env files.../, output
      # deploy
      assert_match /Acquiring the deploy lock/, output
      assert_match /Log into image registry/, output
      assert_match /Pull app image/, output
      assert_match /Ensure Traefik is running/, output
-      assert_match /Ensure app can pass healthcheck/, output
      assert_match /Detect stale containers/, output
      assert_match /Prune old containers and images/, output
      assert_match /Releasing the deploy lock/, output
@@ -51,7 +49,6 @@ class CliMainTest < CliTestCase
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
@@ -65,7 +62,6 @@ class CliMainTest < CliTestCase
      assert_match /Build and push app image/, output
      assert_hook_ran "pre-deploy", output, **hook_variables
      assert_match /Ensure Traefik is running/, output
-      assert_match /Ensure app can pass healthcheck/, output
      assert_match /Detect stale containers/, output
      assert_match /Prune old containers and images/, output
      assert_hook_ran "post-deploy", output, **hook_variables, runtime: true
@@ -78,7 +74,6 @@ class CliMainTest < CliTestCase
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
@@ -88,7 +83,6 @@ class CliMainTest < CliTestCase
      assert_match /Log into image registry/, output
      assert_match /Pull app image/, output
      assert_match /Ensure Traefik is running/, output
-      assert_match /Ensure app can pass healthcheck/, output
      assert_match /Detect stale containers/, output
      assert_match /Prune old containers and images/, output
      assert_match /Releasing the deploy lock/, output
@@ -98,6 +92,9 @@ class CliMainTest < CliTestCase
  test "deploy when locked" do
    Thread.report_on_exception = false

+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+    Dir.stubs(:chdir)
+
    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
      .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }

@@ -111,6 +108,14 @@ class CliMainTest < CliTestCase
    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_debug)
      .with(:stat, ".kamal/locks/app", ">", "/dev/null", "&&", :cat, ".kamal/locks/app/details", "|", :base64, "-d")

+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:git, "-C", anything, :"rev-parse", :HEAD)
+      .returns(Kamal::Git.revision)
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:git, "-C", anything, :status, "--porcelain")
+      .returns("")
+
    assert_raises(Kamal::Cli::LockError) do
      run_command("deploy")
    end
@@ -119,6 +124,9 @@ class CliMainTest < CliTestCase
  test "deploy error when locking" do
    Thread.report_on_exception = false

+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+    Dir.stubs(:chdir)
+
    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
      .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }

@@ -129,6 +137,14 @@ class CliMainTest < CliTestCase
      .with { |*arg| arg[0..1] == [ :mkdir, ".kamal/locks/app" ] }
      .raises(SocketError, "getaddrinfo: nodename nor servname provided, or not known")

+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:git, "-C", anything, :"rev-parse", :HEAD)
+      .returns(Kamal::Git.revision)
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:git, "-C", anything, :status, "--porcelain")
+      .returns("")
+
    assert_raises(SSHKit::Runner::ExecuteError) do
      run_command("deploy")
    end
@@ -154,7 +170,6 @@ class CliMainTest < CliTestCase
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
@@ -185,7 +200,6 @@ class CliMainTest < CliTestCase
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
@@ -197,7 +211,6 @@ class CliMainTest < CliTestCase
    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true }

    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)

@@ -210,7 +223,6 @@ class CliMainTest < CliTestCase
      assert_match /Build and push app image/, output
      assert_hook_ran "pre-deploy", output, **hook_variables
      assert_match /Running the pre-deploy hook.../, output
-      assert_match /Ensure app can pass healthcheck/, output
      assert_hook_ran "post-deploy", output, **hook_variables, runtime: true
    end
  end
@@ -219,13 +231,11 @@ class CliMainTest < CliTestCase
    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }

    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)

    run_command("redeploy", "--skip_push").tap do |output|
      assert_match /Pull app image/, output
-      assert_match /Ensure app can pass healthcheck/, output
    end
  end

@@ -427,6 +437,7 @@ class CliMainTest < CliTestCase
  end

  test "envify" do
+    Pathname.any_instance.expects(:exist?).returns(true).times(3)
    File.expects(:read).with(".env.erb").returns("HELLO=<%= 'world' %>")
    File.expects(:write).with(".env", "HELLO=world", perm: 0600)

@@ -441,6 +452,7 @@ class CliMainTest < CliTestCase
      <% end  -%>
    EOF

+    Pathname.any_instance.expects(:exist?).returns(true).times(3)
    File.expects(:read).with(".env.erb").returns(file.strip)
    File.expects(:write).with(".env", "HELLO=world\nKEY=value\n", perm: 0600)

@@ -448,6 +460,7 @@ class CliMainTest < CliTestCase
  end

  test "envify with destination" do
+    Pathname.any_instance.expects(:exist?).returns(true).times(4)
    File.expects(:read).with(".env.world.erb").returns("HELLO=<%= 'world' %>")
    File.expects(:write).with(".env.world", "HELLO=world", perm: 0600)

@@ -455,6 +468,7 @@ class CliMainTest < CliTestCase
  end

  test "envify with skip_push" do
+    Pathname.any_instance.expects(:exist?).returns(true).times(1)
    File.expects(:read).with(".env.erb").returns("HELLO=<%= 'world' %>")
    File.expects(:write).with(".env", "HELLO=world", perm: 0600)

--- a/test/cli/server_test.rb
+++ b/test/cli/server_test.rb
@@ -1,14 +1,30 @@
 require_relative "cli_test_case"

 class CliServerTest < CliTestCase
+  test "running a command with exec" do
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture)
+    .with("date", verbosity: 1)
+    .returns("Today")
+
+    hosts = "1.1.1.1".."1.1.1.4"
+    run_command("exec", "date").tap do |output|
+      hosts.map do |host|
+        assert_match "Running 'date' on #{hosts.to_a.join(', ')}...", output
+        assert_match "App Host: #{host}\nToday", output
+      end
+    end
+  end
+
  test "bootstrap already installed" do
+    stub_setup
    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(true).at_least_once
    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once

-    assert_equal "", run_command("bootstrap")
+    assert_equal "Acquiring the deploy lock...\nReleasing the deploy lock...", run_command("bootstrap")
  end

  test "bootstrap install as non-root user" do
+    stub_setup
    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', raise_on_non_zero_exit: false).returns(false).at_least_once
    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
@@ -19,11 +35,13 @@ class CliServerTest < CliTestCase
  end

  test "bootstrap install as root user" do
+    stub_setup
    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', raise_on_non_zero_exit: false).returns(true).at_least_once
    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:sh, "-c", "'curl -fsSL https://get.docker.com || wget -O - https://get.docker.com || echo \"exit 1\"'", "|", :sh).at_least_once
    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(".kamal/hooks/pre-connect", anything).at_least_once
    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(".kamal/hooks/docker-setup", anything).at_least_once

    run_command("bootstrap").tap do |output|
--- a/test/commander_test.rb
+++ b/test/commander_test.rb
@@ -99,6 +99,11 @@ class CommanderTest < ActiveSupport::TestCase
    assert_equal [ "workers" ], @kamal.roles_on("1.1.1.3").map(&:name)
  end

+  test "roles_on web comes first" do
+    configure_with(:deploy_with_two_roles_one_host)
+    assert_equal [ "web", "workers" ], @kamal.roles_on("1.1.1.1").map(&:name)
+  end
+
  test "default group strategy" do
    assert_empty @kamal.boot_strategy
  end
--- a/test/commands/app_test.rb
+++ b/test/commands/app_test.rb
@@ -60,7 +60,7 @@ class CommandsAppTest < ActiveSupport::TestCase
    @config[:servers] = { "web" => [ "1.1.1.1" ], "jobs" => { "hosts" => [ "1.1.1.2" ], "cmd" => "bin/jobs", "options" => { "mount" => "somewhere", "cap-add" => true } } }
    assert_equal \
      "docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-jobs.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --label destination --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
-      new_command(role: "jobs").run.join(" ")
+      new_command(role: "jobs", host: "1.1.1.2").run.join(" ")
  end

  test "run with logging config" do
@@ -80,6 +80,15 @@ class CommandsAppTest < ActiveSupport::TestCase
      new_command.run.join(" ")
  end

+  test "run with tags" do
+    @config[:servers] = [ { "1.1.1.1" => "tag1" } ]
+    @config[:env]["tags"] = { "tag1" => { "ENV1" => "value1" } }
+
+    assert_equal \
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      new_command.run.join(" ")
+  end
+
  test "start" do
    assert_equal \
      "docker start app-web-999",
@@ -183,6 +192,15 @@ class CommandsAppTest < ActiveSupport::TestCase
      new_command.execute_in_new_container("bin/rails", "db:setup", env: { "foo" => "bar" }).join(" ")
  end

+  test "execute in new container with tags" do
+    @config[:servers] = [ { "1.1.1.1" => "tag1" } ]
+    @config[:env]["tags"] = { "tag1" => { "ENV1" => "value1" } }
+
+    assert_equal \
+      "docker run --rm --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" dhh/app:999 bin/rails db:setup",
+      new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ")
+  end
+
  test "execute in new container with custom options" do
    @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } }
    assert_equal \
@@ -204,18 +222,26 @@ class CommandsAppTest < ActiveSupport::TestCase

  test "execute in new container over ssh" do
    assert_match %r{docker run -it --rm --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails c},
-      new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1", env: {})
+      new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {})
+  end
+
+  test "execute in new container over ssh with tags" do
+    @config[:servers] = [ { "1.1.1.1" => "tag1" } ]
+    @config[:env]["tags"] = { "tag1" => { "ENV1" => "value1" } }
+
+    assert_equal "ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" dhh/app:999 bin/rails c'",
+      new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {})
  end

  test "execute in new container with custom options over ssh" do
    @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } }
    assert_match %r{docker run -it --rm --env-file .kamal/env/roles/app-web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails c},
-      new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1", env: {})
+      new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {})
  end

  test "execute in existing container over ssh" do
    assert_match %r{docker exec -it app-web-999 bin/rails c},
-      new_command.execute_in_existing_container_over_ssh("bin/rails", "c", host: "app-1", env: {})
+      new_command.execute_in_existing_container_over_ssh("bin/rails", "c", env: {})
  end

  test "run over ssh" do
@@ -418,8 +444,8 @@ class CommandsAppTest < ActiveSupport::TestCase
  end

  private
-    def new_command(role: "web", **additional_config)
+    def new_command(role: "web", host: "1.1.1.1", **additional_config)
      config = Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999")
-      Kamal::Commands::App.new(config, role: config.role(role))
+      Kamal::Commands::App.new(config, role: config.role(role), host: host)
    end
 end
--- a/test/commands/builder_test.rb
+++ b/test/commands/builder_test.rb
@@ -9,7 +9,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "cache" => { "type" => "gha" } })
    assert_equal "multiarch", builder.name
    assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
      builder.push.join(" ")
  end

@@ -17,7 +17,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "multiarch" => false })
    assert_equal "native", builder.name
    assert_equal \
-      "git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
      builder.push.join(" ")
  end

@@ -25,7 +25,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "multiarch" => false, "cache" => { "type" => "gha" } })
    assert_equal "native/cached", builder.name
    assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
      builder.push.join(" ")
  end

@@ -33,7 +33,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "local" => {}, "remote" => {}, "cache" => { "type" => "gha" } })
    assert_equal "multiarch/remote", builder.name
    assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
      builder.push.join(" ")
  end

@@ -41,7 +41,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "local" => { "arch" => "amd64" } })
    assert_equal "multiarch", builder.name
    assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .",
      builder.push.join(" ")
  end

@@ -49,7 +49,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "remote" => { "arch" => "amd64" }, "cache" => { "type" => "gha" } })
    assert_equal "native/remote", builder.name
    assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64 --builder kamal-app-native-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64 --builder kamal-app-native-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
      builder.push.join(" ")
  end

@@ -83,6 +83,13 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    end
  end

+  test "build target" do
+    builder = new_builder_command(builder: { "target" => "prod" })
+    assert_equal \
+      "-t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --target prod",
+      builder.target.build_options.join(" ")
+  end
+
  test "build context" do
    builder = new_builder_command(builder: { "context" => ".." })
    assert_equal \
@@ -93,21 +100,21 @@ class CommandsBuilderTest < ActiveSupport::TestCase
  test "native push with build args" do
    builder = new_builder_command(builder: { "multiarch" => false, "args" => { "a" => 1, "b" => 2 } })
    assert_equal \
-      "git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
      builder.push.join(" ")
  end

  test "multiarch push with build args" do
    builder = new_builder_command(builder: { "args" => { "a" => 1, "b" => 2 } })
    assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile .",
      builder.push.join(" ")
  end

  test "native push with build secrets" do
    builder = new_builder_command(builder: { "multiarch" => false, "secrets" => [ "a", "b" ] })
    assert_equal \
-      "git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
      builder.push.join(" ")
  end

@@ -155,4 +162,8 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    def new_builder_command(additional_config = {})
      Kamal::Commands::Builder.new(Kamal::Configuration.new(@config.merge(additional_config), version: "123"))
    end
+
+    def build_directory
+      "#{Dir.tmpdir}/kamal-clones/app/kamal/"
+    end
 end
--- a/test/commands/healthcheck_test.rb
+++ b/test/commands/healthcheck_test.rb
@@ -1,114 +0,0 @@
-require "test_helper"
-
-class CommandsHealthcheckTest < ActiveSupport::TestCase
-  setup do
-    @config = {
-      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
-      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
-    }
-  end
-
-  test "run" do
-    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "run with custom port" do
-    @config[:healthcheck] = { "port" => 3001 }
-
-    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3001 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3001/up || exit 1\" --health-interval \"1s\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "run with destination" do
-    @destination = "staging"
-
-    assert_equal \
-      "docker run --detach --name healthcheck-app-staging-123 --publish 3999:3000 --label service=healthcheck-app-staging -e KAMAL_CONTAINER_NAME=\"healthcheck-app-staging\" --env-file .kamal/env/roles/app-web-staging.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "run with custom healthcheck" do
-    @config[:healthcheck] = { "cmd" => "/bin/up" }
-
-    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/up\" --health-interval \"1s\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "run with custom options" do
-    @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere" } } }
-    @config[:healthcheck] = { "exposed_port" => 4999 }
-    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 4999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "status" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'",
-      new_command.status.join(" ")
-  end
-
-  test "container_health_log" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker inspect --format '{{json .State.Health}}'",
-      new_command.container_health_log.join(" ")
-  end
-
-  test "stop" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker stop",
-      new_command.stop.join(" ")
-  end
-
-  test "stop with destination" do
-    @destination = "staging"
-
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-staging-123$ --quiet | xargs docker stop",
-      new_command.stop.join(" ")
-  end
-
-  test "remove" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker container rm",
-      new_command.remove.join(" ")
-  end
-
-  test "remove with destination" do
-    @destination = "staging"
-
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-staging-123$ --quiet | xargs docker container rm",
-      new_command.remove.join(" ")
-  end
-
-  test "logs" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker logs --tail 50 2>&1",
-      new_command.logs.join(" ")
-  end
-
-  test "logs with custom lines number" do
-    @config[:healthcheck] = { "log_lines" => 150 }
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker logs --tail 150 2>&1",
-      new_command.logs.join(" ")
-  end
-
-  test "logs with destination" do
-    @destination = "staging"
-
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-staging-123$ --quiet | xargs docker logs --tail 50 2>&1",
-      new_command.logs.join(" ")
-  end
-
-  private
-    def new_command
-      Kamal::Commands::Healthcheck.new(Kamal::Configuration.new(@config, destination: @destination, version: "123"))
-    end
-end
--- a/test/configuration/builder_test.rb
+++ b/test/configuration/builder_test.rb
@@ -140,7 +140,7 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase
  end

  test "context" do
-    assert_equal "-", @config.builder.context
+    assert_equal ".", @config.builder.context
  end

  test "setting context" do
--- a/test/configuration/env/tags_test.rb
+++ b/test/configuration/env/tags_test.rb
@@ -0,0 +1,112 @@
+require "test_helper"
+
+class ConfigurationEnvTagsTest < ActiveSupport::TestCase
+  setup do
+    @deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+      servers: [ { "1.1.1.1" => "odd" }, { "1.1.1.2" => "even" }, { "1.1.1.3" => [ "odd", "three" ] } ],
+      env: {
+        "clear" => { "REDIS_URL" => "redis://x/y", "THREE" => "false" },
+        "tags" => {
+          "odd" => { "TYPE" => "odd" },
+          "even" => { "TYPE" => "even" },
+          "three" => { "THREE" => "true" }
+        }
+      }
+    }
+
+    @config = Kamal::Configuration.new(@deploy)
+
+    @deploy_with_roles = @deploy.dup.merge({
+      servers: {
+        "web" => [ { "1.1.1.1" => "odd" }, "1.1.1.2" ],
+        "workers" => {
+          "hosts" => [ { "1.1.1.3" => [ "odd", "oddjob" ] }, "1.1.1.4" ],
+          "cmd" => "bin/jobs",
+          "env" => {
+            "REDIS_URL" => "redis://a/b",
+            "WEB_CONCURRENCY" => 4
+          }
+        }
+      },
+      env: {
+        "tags" => {
+          "odd" => { "TYPE" => "odd" },
+          "oddjob" => { "TYPE" => "oddjob" }
+        }
+      }
+    })
+
+    @config_with_roles = Kamal::Configuration.new(@deploy_with_roles)
+  end
+
+  test "tags" do
+    assert_equal 3, @config.env_tags.size
+    assert_equal %w[ odd even three ], @config.env_tags.map(&:name)
+    assert_equal({ "TYPE" => "odd" }, @config.env_tag("odd").env.clear)
+    assert_equal({ "TYPE" => "even" }, @config.env_tag("even").env.clear)
+    assert_equal({ "THREE" => "true" }, @config.env_tag("three").env.clear)
+  end
+
+  test "tags with roles" do
+    assert_equal 2, @config_with_roles.env_tags.size
+    assert_equal %w[ odd oddjob ], @config_with_roles.env_tags.map(&:name)
+    assert_equal({ "TYPE" => "odd" }, @config_with_roles.env_tag("odd").env.clear)
+    assert_equal({ "TYPE" => "oddjob" }, @config_with_roles.env_tag("oddjob").env.clear)
+  end
+
+  test "tag overrides env" do
+    assert_equal "false", @config.role("web").env("1.1.1.1").clear["THREE"]
+    assert_equal "true", @config.role("web").env("1.1.1.3").clear["THREE"]
+  end
+
+  test "later tag wins" do
+    deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+      servers: [ { "1.1.1.1" => [ "first", "second" ] } ],
+      env: {
+        "tags" => {
+          "first" => { "TYPE" => "first" },
+          "second" => { "TYPE" => "second" }
+        }
+      }
+    }
+
+    config = Kamal::Configuration.new(deploy)
+    assert_equal "second", config.role("web").env("1.1.1.1").clear["TYPE"]
+  end
+
+  test "tag secret env" do
+    ENV["PASSWORD"] = "hello"
+
+    deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+      servers: [ { "1.1.1.1" => "secrets" } ],
+      env: {
+        "tags" => {
+          "secrets" => { "secret" => [ "PASSWORD" ] }
+        }
+      }
+    }
+
+    config = Kamal::Configuration.new(deploy)
+    assert_equal "hello", config.role("web").env("1.1.1.1").secrets["PASSWORD"]
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "tag clear env" do
+    deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+      servers: [ { "1.1.1.1" => "clearly" } ],
+      env: {
+        "tags" => {
+          "clearly" => { "clear" => { "FOO" => "bar" } }
+        }
+      }
+    }
+
+    config = Kamal::Configuration.new(deploy)
+    assert_equal "bar", config.role("web").env("1.1.1.1").clear["FOO"]
+  end
+end
--- a/test/configuration/role_test.rb
+++ b/test/configuration/role_test.rb
@@ -70,10 +70,10 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
  end

  test "env overwritten by role" do
-    assert_equal "redis://a/b", @config_with_roles.role(:workers).env.clear["REDIS_URL"]
+    assert_equal "redis://a/b", @config_with_roles.role(:workers).env("1.1.1.3").clear["REDIS_URL"]

-    assert_equal "\n", @config_with_roles.role(:workers).env.secrets_io.string
-    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
+    assert_equal "\n", @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
  end

  test "container name" do
@@ -86,7 +86,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
  end

  test "env args" do
-    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
  end

  test "env secret overwritten by role" do
@@ -117,8 +117,8 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
      DB_PASSWORD=secret&\"123
    ENV

-    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env.secrets_io.string
-    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
+    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
  ensure
    ENV["REDIS_PASSWORD"] = nil
    ENV["DB_PASSWORD"] = nil
@@ -141,8 +141,8 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
      DB_PASSWORD=secret123
    ENV

-    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env.secrets_io.string
-    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
+    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
  ensure
    ENV["DB_PASSWORD"] = nil
  end
@@ -163,8 +163,8 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
      REDIS_PASSWORD=secret456
    ENV

-    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env.secrets_io.string
-    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
+    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
  ensure
    ENV["REDIS_PASSWORD"] = nil
  end
@@ -191,14 +191,14 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
      REDIS_PASSWORD=secret456
    ENV

-    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env.secrets_io.string
-    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://c/d\"" ], @config_with_roles.role(:workers).env_args
+    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://c/d\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
  ensure
    ENV["REDIS_PASSWORD"] = nil
  end

  test "env secrets_file" do
-    assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).env.secrets_file
+    assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).env("1.1.1.3").secrets_file
  end

  test "uses cord" do
--- a/test/configuration_test.rb
+++ b/test/configuration_test.rb
@@ -272,7 +272,7 @@ class ConfigurationTest < ActiveSupport::TestCase
        volume_args: [ "--volume", "/local/path:/container/path" ],
        builder: {},
        logging: [ "--log-opt", "max-size=\"10m\"" ],
-        healthcheck: { "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 } }
+        healthcheck: { "path"=>"/up", "port"=>3000, "max_attempts" => 7, "cord" => "/tmp/kamal-cord", "log_lines" => 50 } }

    assert_equal expected_config, @config.to_h
  end
--- a/test/fixtures/deploy_with_env_tags.yml
+++ b/test/fixtures/deploy_with_env_tags.yml
@@ -0,0 +1,28 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - 1.1.1.1: site1
+    - 1.1.1.2: [ site1 experimental ]
+    - 1.2.1.1: site2
+    - 1.2.1.2: site2
+  workers:
+    - 1.1.1.3: site1
+    - 1.1.1.4: site1
+    - 1.2.1.3: site2
+    - 1.2.1.4: [ site2 experimental ]
+env:
+  clear:
+    TEST: "root"
+    EXPERIMENT: "disabled"
+  tags:
+    site1:
+      SITE: site1
+    site2:
+      SITE: site2
+    experimental:
+      EXPERIMENT: "enabled"
+
+registry:
+  username: user
+  password: pw
--- a/test/fixtures/deploy_with_remote_builder_and_custom_ports.yml
+++ b/test/fixtures/deploy_with_remote_builder_and_custom_ports.yml
@@ -0,0 +1,45 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+registry:
+  username: user
+  password: pw
+
+accessories:
+  mysql:
+    image: mysql:5.7
+    host: 1.1.1.3
+    port: 3306
+    env:
+      clear:
+        MYSQL_ROOT_HOST: '%'
+      secret:
+        - MYSQL_ROOT_PASSWORD
+    files:
+      - test/fixtures/files/my.cnf:/etc/mysql/my.cnf
+    directories:
+      - data:/var/lib/mysql
+  redis:
+    image: redis:latest
+    roles:
+      - web
+    port: 6379
+    directories:
+      - data:/data
+
+readiness_delay: 0
+
+ssh:
+  user: root
+  port: 22
+
+builder:
+  remote:
+    arch: amd64
+    host: ssh://app@1.1.1.5:2122
--- a/test/fixtures/deploy_with_two_roles_one_host.yml
+++ b/test/fixtures/deploy_with_two_roles_one_host.yml
@@ -0,0 +1,15 @@
+service: app
+image: dhh/app
+servers:
+  workers:
+    hosts:
+      - 1.1.1.1
+  web:
+    hosts:
+      - 1.1.1.1
+env:
+  REDIS_URL: redis://x/y
+registry:
+  server: registry.digitalocean.com
+  username: user
+  password: pw
--- a/test/fixtures/deploy_with_uncommon_hostnames.yml
+++ b/test/fixtures/deploy_with_uncommon_hostnames.yml
@@ -0,0 +1,8 @@
+service: app
+image: dhh/app
+servers:
+  - "this-hostname-with-random-part-is-too-long.example.com"
+  - "this-hostname-is-really-unacceptably-long-to-be-honest.example.com"
+registry:
+  username: user
+  password: pw
--- a/test/fixtures/deploy_without_clone.yml
+++ b/test/fixtures/deploy_without_clone.yml
@@ -0,0 +1,39 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+registry:
+  username: user
+  password: pw
+
+accessories:
+  mysql:
+    image: mysql:5.7
+    host: 1.1.1.3
+    port: 3306
+    env:
+      clear:
+        MYSQL_ROOT_HOST: '%'
+      secret:
+        - MYSQL_ROOT_PASSWORD
+    files:
+      - test/fixtures/files/my.cnf:/etc/mysql/my.cnf
+    directories:
+      - data:/var/lib/mysql
+  redis:
+    image: redis:latest
+    roles:
+      - web
+    port: 6379
+    directories:
+      - data:/data
+
+readiness_delay: 0
+
+builder:
+  context: "."
--- a/test/integration/broken_deploy_test.rb
+++ b/test/integration/broken_deploy_test.rb
@@ -0,0 +1,34 @@
+require_relative "integration_test"
+
+class BrokenDeployTest < IntegrationTest
+  test "deploying a bad image" do
+    @app = "app_with_roles"
+
+    kamal :envify
+
+    first_version = latest_app_version
+
+    kamal :deploy
+
+    assert_app_is_up version: first_version
+    assert_container_running host: :vm3, name: "app-workers-#{first_version}"
+
+    second_version = break_app
+
+    output = kamal :deploy, raise_on_error: false, capture: true
+
+    assert_failed_deploy output
+    assert_app_is_up version: first_version
+    assert_container_running host: :vm3, name: "app-workers-#{first_version}"
+    assert_container_not_running host: :vm3, name: "app-workers-#{second_version}"
+  end
+
+  private
+    def assert_failed_deploy(output)
+      assert_match "Waiting for the first healthy web container before booting workers on vm3...", output
+      assert_match /First web container is unhealthy on vm[12], not booting other roles/, output
+      assert_match "First web container is unhealthy, not booting workers on vm3", output
+      assert_match "nginx: [emerg] unexpected end of file, expecting \";\" or \"}\" in /etc/nginx/conf.d/default.conf:2", output
+      assert_match 'ERROR {"Status":"unhealthy","FailingStreak":0,"Log":[]}', output
+    end
+end
--- a/test/integration/docker/deployer/app/.env.erb
+++ b/test/integration/docker/deployer/app/.env.erb
@@ -1 +1,2 @@
 SECRET_TOKEN='1234 with "中文"'
+SECRET_TAG='TAGME'
--- a/test/integration/docker/deployer/app/config/deploy.yml
+++ b/test/integration/docker/deployer/app/config/deploy.yml
@@ -2,13 +2,20 @@ service: app
 image: app
 servers:
  - vm1
-  - vm2
+  - vm2: [ tag1, tag2 ]
 env:
  clear:
    CLEAR_TOKEN: 4321
+    CLEAR_TAG: ""
    HOST_TOKEN: "${HOST_TOKEN}"
  secret:
    - SECRET_TOKEN
+  tags:
+    tag1:
+      CLEAR_TAG: tagged
+    tag2:
+      secret:
+        - SECRET_TAG
 asset_path: /usr/share/nginx/html/versions

 registry:
@@ -21,6 +28,7 @@ builder:
    COMMIT_SHA: <%= `git rev-parse HEAD` %>
 healthcheck:
  cmd: wget -qO- http://localhost > /dev/null || exit 1
+  max_attempts: 3
 traefik:
  args:
    accesslog: true
@@ -34,3 +42,4 @@ accessories:
    roles:
      - web
 stop_wait_time: 1
+readiness_delay: 0
--- a/test/integration/docker/deployer/app_with_roles/config/deploy.yml
+++ b/test/integration/docker/deployer/app_with_roles/config/deploy.yml
@@ -22,6 +22,7 @@ builder:
    COMMIT_SHA: <%= `git rev-parse HEAD` %>
 healthcheck:
  cmd: wget -qO- http://localhost > /dev/null || exit 1
+  max_attempts: 3
 traefik:
  args:
    accesslog: true
@@ -35,3 +36,4 @@ accessories:
    roles:
      - web
 stop_wait_time: 1
+readiness_delay: 0
--- a/test/integration/docker/deployer/break_app.sh
+++ b/test/integration/docker/deployer/break_app.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+cd $1 && echo "bad nginx config" > default.conf && git commit -am 'Broken'
--- a/test/integration/integration_test.rb
+++ b/test/integration/integration_test.rb
@@ -78,6 +78,11 @@ class IntegrationTest < ActiveSupport::TestCase
      latest_app_version
    end

+    def break_app
+      deployer_exec "./break_app.sh #{@app}", workdir: "/"
+      latest_app_version
+    end
+
    def latest_app_version
      deployer_exec("git rev-parse HEAD", capture: true)
    end
@@ -131,4 +136,16 @@ class IntegrationTest < ActiveSupport::TestCase
        puts "Tried to get the response code again and got #{app_response.code}"
      end
    end
+
+    def assert_container_running(host:, name:)
+      assert container_running?(host: host, name: name)
+    end
+
+    def assert_container_not_running(host:, name:)
+      assert_not container_running?(host: host, name: name)
+    end
+
+    def container_running?(host:, name:)
+      docker_compose("exec #{host} docker ps --filter=name=#{name} | tail -n+2", capture: true).tap { |x| p [ x, x.strip, x.strip.present? ] }.strip.present?
+    end
 end
--- a/test/integration/main_test.rb
+++ b/test/integration/main_test.rb
@@ -3,8 +3,7 @@ require_relative "integration_test"
 class MainTest < IntegrationTest
  test "envify, deploy, redeploy, rollback, details and audit" do
    kamal :envify
-    assert_local_env_file "SECRET_TOKEN='1234 with \"中文\"'"
-    assert_remote_env_file "SECRET_TOKEN=1234 with \"中文\""
+    assert_env_files
    remove_local_env_file

    first_version = latest_app_version
@@ -14,9 +13,7 @@ class MainTest < IntegrationTest
    kamal :deploy
    assert_app_is_up version: first_version
    assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy"
-    assert_env :CLEAR_TOKEN, "4321", version: first_version
-    assert_env :HOST_TOKEN, "abcd", version: first_version
-    assert_env :SECRET_TOKEN, "1234 with \"中文\"", version: first_version
+    assert_envs version: first_version

    second_version = update_app_rev

@@ -59,6 +56,12 @@ class MainTest < IntegrationTest
    assert_app_is_up version: version
    assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy"
    assert_container_running host: :vm3, name: "app-workers-#{version}"
+
+    second_version = update_app_rev
+
+    kamal :redeploy
+    assert_app_is_up version: second_version
+    assert_container_running host: :vm3, name: "app-workers-#{second_version}"
  end

  test "config" do
@@ -76,7 +79,7 @@ class MainTest < IntegrationTest
    assert_equal({ user: "root", port: 22, keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])
    assert_equal({ "multiarch" => false, "args" => { "COMMIT_SHA" => version } }, config[:builder])
    assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging]
-    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord"=>"/tmp/kamal-cord", "log_lines" => 50, "cmd"=>"wget -qO- http://localhost > /dev/null || exit 1" }, config[:healthcheck])
+    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 3, "cord"=>"/tmp/kamal-cord", "log_lines" => 50, "cmd"=>"wget -qO- http://localhost > /dev/null || exit 1" }, config[:healthcheck])
  end

  test "setup and remove" do
@@ -97,16 +100,38 @@ class MainTest < IntegrationTest
      assert_equal contents, deployer_exec("cat .env", capture: true)
    end

-    def assert_env(key, value, version:)
-      assert_equal "#{key}=#{value}", docker_compose("exec vm1 docker exec app-web-#{version} env | grep #{key}", capture: true)
+    def assert_envs(version:)
+      assert_env :CLEAR_TOKEN, "4321", version: version, vm: :vm1
+      assert_env :HOST_TOKEN, "abcd", version: version, vm: :vm1
+      assert_env :SECRET_TOKEN, "1234 with \"中文\"", version: version, vm: :vm1
+      assert_no_env :CLEAR_TAG, version: version, vm: :vm1
+      assert_no_env :SECRET_TAG, version: version, vm: :vm11
+      assert_env :CLEAR_TAG, "tagged", version: version, vm: :vm2
+      assert_env :SECRET_TAG, "TAGME", version: version, vm: :vm2
+    end
+
+    def assert_env(key, value, vm:, version:)
+      assert_equal "#{key}=#{value}", docker_compose("exec #{vm} docker exec app-web-#{version} env | grep #{key}", capture: true)
+    end
+
+    def assert_no_env(key, vm:, version:)
+      assert_raises(RuntimeError, /exit 1/) do
+        docker_compose("exec #{vm} docker exec app-web-#{version} env | grep #{key}", capture: true)
+      end
+    end
+
+    def assert_env_files
+      assert_local_env_file "SECRET_TOKEN='1234 with \"中文\"'\nSECRET_TAG='TAGME'"
+      assert_remote_env_file "SECRET_TOKEN=1234 with \"中文\"", vm: :vm1
+      assert_remote_env_file "SECRET_TOKEN=1234 with \"中文\"\nSECRET_TAG=TAGME", vm: :vm2
    end

    def remove_local_env_file
      deployer_exec("rm .env")
    end

-    def assert_remote_env_file(contents)
-      assert_equal contents, docker_compose("exec vm1 cat /root/.kamal/env/roles/app-web.env", capture: true)
+    def assert_remote_env_file(contents, vm:)
+      assert_equal contents, docker_compose("exec #{vm} cat /root/.kamal/env/roles/app-web.env", capture: true)
    end

    def assert_no_remote_env_file
@@ -138,8 +163,4 @@ class MainTest < IntegrationTest
      assert vm1_image_ids.any?
      assert vm1_container_ids.any?
    end
-
-    def assert_container_running(host:, name:)
-      assert docker_compose("exec #{host} docker ps --filter=name=#{name} -q", capture: true).strip.present?
-    end
 end
Author	SHA1	Message	Date
Donal McBreen	6e60ab918a	Bump version for 1.6.0	2024-06-03 08:34:12 +01:00
Donal McBreen	90ecb6a12a	Merge pull request #821 from basecamp/retry-clone Handle corrupt git clones	2024-05-28 15:23:36 +01:00
Donal McBreen	2c2053558a	Handle corrupt git clones When cloning the git repo: 1. Try to clone 2. If there's already a build directory reset it 3. Check the clone is valid If anything goes wrong during that process: 1. Delete the clone directory 2. Clone it again 3. Check the clone is valid Raise any errors after that	2024-05-27 11:17:34 +01:00
Donal McBreen	10b8c826d8	Merge pull request #755 from basecamp/lock-less Lock less	2024-05-21 12:33:45 +01:00
Donal McBreen	187861fa60	Space not tab	2024-05-21 12:20:19 +01:00
Donal McBreen	5ff1203c80	Always lock before pre-deploy hook	2024-05-21 12:02:16 +01:00
Donal McBreen	0e73f02743	Split lock and connection setup Allow run the pre-connect hook before the first SSH command is executed, but only run the locking in `with_lock` blocks.	2024-05-21 12:02:16 +01:00
Donal McBreen	83d0078525	Confirm outside mutating	2024-05-21 12:02:16 +01:00
Donal McBreen	96ef0fbc4d	Fix merge error	2024-05-21 12:02:16 +01:00
Donal McBreen	b12654ccd0	Don't lock until confirmed	2024-05-21 12:02:16 +01:00
Donal McBreen	64f5955444	Don't hold lock on error	2024-05-21 12:02:12 +01:00
Donal McBreen	d2a719998a	Building doesn't need a deploy lock	2024-05-21 12:01:08 +01:00
Donal McBreen	6a7c90cf4d	Only stopping containers locks	2024-05-21 12:01:07 +01:00
Donal McBreen	2c2d94c6d9	Merge pull request #740 from basecamp/remove-healthcheck-step Remove the healthcheck step	2024-05-21 12:00:25 +01:00
Donal McBreen	c62bd1dc31	Merge pull request #815 from basecamp/envify-already-pushes-env Envify already env pushes	2024-05-21 11:57:55 +01:00
Donal McBreen	a83df9e135	Merge pull request #620 from mlitwiniuk/allow_custom_ports_for_ssh Allow custom user and port for builder host	2024-05-21 11:49:06 +01:00
Donal McBreen	7b55f4734e	Envify already env pushes `kamal envify` will do `kamal env push` for us, so no need to call it ourselves during setup.	2024-05-21 11:47:51 +01:00
Donal McBreen	1e296c4140	Update sshkit_with_ext.rb	2024-05-21 11:38:30 +01:00
Donal McBreen	9700e2b3c4	Merge pull request #646 from nickhammond/server/exec Add in a server exec command for running ad-hoc commands directly on the server	2024-05-21 11:34:20 +01:00
Donal McBreen	706b82baa1	Simplify messages and remove multiple execute error	2024-05-21 10:40:01 +01:00
Donal McBreen	fa7e941648	Make SSHKit::Runner::Parallel fail slow Using a different SSHKit runner doesn't work well, because the group runner uses the Parallel runner internally. So instead we'll patch its behaviour to fail slow. We'll also get it to return all the errors so we can report on all the hosts that failed.	2024-05-21 09:33:22 +01:00
Donal McBreen	78c0a0ba4b	Don't start other roles we have a healthy container If a primary role container is unhealthy, we might take a while to timeout the health check poller. In the meantime if we have started the other roles, they'll be running tow containers. This could be a problem, especially if they read run jobs as that doubles the worker capacity which could cause exessive load. We'll wait for the first primary role container to boot successfully before starting the other containers from other roles.	2024-05-21 08:37:36 +01:00
Donal McBreen	060e5d2027	Update lib/kamal/cli/server.rb Co-authored-by: Sijawusz Pur Rahnama <sija@sija.pl>	2024-05-21 08:22:20 +01:00
Nick Hammond	8a4f7163bb	Apply suggestions from code review Co-authored-by: Donal McBreen <dmcbreen@gmail.com>	2024-05-20 11:15:14 -07:00
Donal McBreen	ee758d951a	Only use barrier when needed, more descriptive info	2024-05-20 12:18:30 +01:00
Donal McBreen	bb2ca81d87	Fix rebase method duplication	2024-05-20 12:18:30 +01:00
Donal McBreen	773ba3a5ab	Show container logs and healthcheck status on failure	2024-05-20 12:18:30 +01:00
Donal McBreen	5be6fa3b4e	Improve comments	2024-05-20 12:18:30 +01:00
Donal McBreen	07c5658396	Remove redundant method	2024-05-20 12:18:30 +01:00
Donal McBreen	0efb5ccfff	Remove the healthcheck step To speed up deployments, we'll remove the healthcheck step. This adds some risk to deployments for non-web roles - if they don't have a Docker healthcheck configured then the only check we do is if the container is running. If there is a bad image we might see the container running before it exits and deploy it. Previously the healthcheck step would have avoided this by ensuring a web container could boot and serve traffic first. To mitigate this, we'll add a deployment barrier. Until one of the primary role containers passes its healthcheck, we'll keep the barrier up and avoid stopping the containers on the non-primary roles. It the primary role container fails its healthcheck, we'll close the barrier and shut down the new containers on the waiting roles. We also have a new integration test to check we correctly handle a a broken image. This highlighted that SSHKit's default runner will stop at the first error it encounters. We'll now have a custom runner that waits for all threads to finish allowing them to clean up.	2024-05-20 12:18:30 +01:00
Donal McBreen	990f1b4413	Merge pull request #798 from basecamp/git-clone Build from within a git clone by default	2024-05-20 12:18:07 +01:00
Donal McBreen	da9428f64d	Merge pull request #813 from basecamp/handle-no-env-tags Don't blow up if there are no env tags	2024-05-20 10:58:11 +01:00
Donal McBreen	17dcaccb6a	Don't blow up if there are no env tags	2024-05-20 10:50:07 +01:00
Donal McBreen	448349d0e5	Merge pull request #812 from basecamp/sshkit-1.22.2-minimum Set sshkit minimum version to 1.22.2	2024-05-20 10:39:17 +01:00
Donal McBreen	b6dba57c7d	Set sshkit minimum version to 1.22.2 This includes a fix for a bug in the eviction thread that could cause this error: ``` [ERROR (IOError): Exception while executing on host foo: closed stream] ``` See https://github.com/capistrano/sshkit/pull/534	2024-05-20 10:06:53 +01:00
Donal McBreen	0ea2a2c509	Don't include destination in clone directory Reusing the clone directory should allow caching of the build context between deployments to different destinations.	2024-05-20 09:34:42 +01:00
Donal McBreen	307750ff70	Build from within a git clone by default Docker does not respect the .dockerignore file when building from a tar. Instead by default we'll make a local clone into a tmp directory and build from there. Subsequent builds will reset the clone to match the checkout. Compared to building directly in the repo, we'll have reproducible builds. Compared to using a git archive: 1. .dockerignore is respected 2. We'll have faster builds - docker can be smarter about caching the build context on subsequent builds from a directory To build from the repo directly, set the build context to "." in the config. If there are uncommitted changes, we'll warn about them either being included or ignored depending on whether we build from the clone.	2024-05-20 09:30:56 +01:00
Donal McBreen	88947b6a7b	Merge pull request #805 from basecamp/env-under-tags Move env_tags under env key	2024-05-15 11:09:47 +01:00
Donal McBreen	f48c227768	Move env_tags under env key Instead of: ``` env: CLEAR_TAG: untagged env_tags: tag1: CLEAR_TAG: tagged ``` We'll have: ``` env: clear: CLEAR_TAG: untagged tags: tag1: CLEAR_TAG: tagged ```	2024-05-15 10:19:22 +01:00
David Heinemeier Hansson	f98380ef0c	Merge pull request #802 from basecamp/envify-during-setup Envify during setup	2024-05-14 16:00:28 -07:00
David Heinemeier Hansson	0bc27c10cc	Fix tests	2024-05-14 11:59:42 -07:00
David Heinemeier Hansson	e58d2f67f2	Fix env template path check and tests	2024-05-14 10:07:31 -07:00
David Heinemeier Hansson	938ac375a1	Only envify if there is a template file available	2024-05-13 17:08:53 -07:00
David Heinemeier Hansson	dc1f707a56	Fix test	2024-05-13 17:01:50 -07:00
David Heinemeier Hansson	033f2a3401	Correct invocation	2024-05-13 16:59:50 -07:00
David Heinemeier Hansson	7cac7e6fb0	Envify during setup	2024-05-13 15:18:11 -07:00
Nick Hammond	fb58fc0ba6	Add in a server exec command for running ad-hoc commands directly on the server	2024-05-13 14:17:06 -07:00
Donal McBreen	12cad5458a	Merge pull request #762 from kryachkov/main Trim long hostnames	2024-05-10 16:05:27 +01:00
Donal McBreen	f8b7f74543	Merge pull request #786 from hundredwatt/add-target-option-to-builder Add --target option to Builder to support multi-stage Docker builds	2024-05-10 15:15:31 +01:00
Donal McBreen	489d6dbcbb	Merge pull request #789 from basecamp/host-tags Host specific env with tags	2024-05-10 08:08:29 +01:00
Donal McBreen	6d062ce271	Host specific env with tags Allow hosts to be tagged so we can have host specific env variables. We might want host specific env variables for things like datacenter specific tags or testing GC settings on a specific host. Right now you either need to set up a separate role, or have the app be host aware. Now you can define tag env variables and assign those to hosts. For example: ``` servers: - 1.1.1.1 - 1.1.1.2: tag1 - 1.1.1.2: tag2 - 1.1.1.3: [ tag1, tag2 ] env_tags: tag1: ENV1: value1 tag2: ENV2: value2 ``` The tag env supports the full env format, allowing you to set secret and clear values.	2024-05-09 16:02:45 +01:00
Jason Nochlin	1e44cc2597	fix rubocop violation	2024-05-08 19:22:25 -06:00
André Falk	63c47eca4c	Trim long hostnames Hostnames longer than 64 characters are not supported by docker	2024-05-07 19:06:39 +02:00
Jason Nochlin	947be0877f	add --target option for builder configuration	2024-04-27 10:24:47 -06:00
Maciej Litwiniuk	2f912367ac	Allow custom user and port for builder host When ssh options are set, they overwrite username and password passed as ssh builder uri. Passing part of uri for ssh-kit is fine, as it then properly extracts username and password and forwards it as host.ssh_options (in which case it's no longer empty)	2024-04-17 17:49:50 +02:00