Bump version for 1.6.0

Handle corrupt git clones

.github/workflows/ci.yml

@@ -5,6 +5,21 @@ on:
       - main
   pull_request:
 jobs:
+  rubocop:
+    name: RuboCop
+    runs-on: ubuntu-latest
+    env:
+      BUNDLE_ONLY: rubocop
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Setup Ruby and install gems
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.3.0
+          bundler-cache: true
+      - name: Run Rubocop
+        run: bundle exec rubocop --parallel
   tests:
     strategy:
       matrix:

.rubocop.yml

@@ -0,0 +1,2 @@
+inherit_gem:
+  rubocop-rails-omakase: rubocop.yml

Gemfile

@@ -1,4 +1,8 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 
 gemspec
+
+group :rubocop do
+  gem "rubocop-rails-omakase", require: false
+end

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kamal (1.4.0)
+    kamal (1.6.0)
       activesupport (>= 7.0)
       base64 (~> 0.2)
       bcrypt_pbkdf (~> 1.0)
@@ -9,7 +9,7 @@ PATH
       dotenv (~> 2.8)
       ed25519 (~> 1.2)
       net-ssh (~> 7.0)
-      sshkit (~> 1.21)
+      sshkit (>= 1.22.2, < 2.0)
       thor (~> 1.2)
       zeitwerk (~> 2.5)
 
@@ -42,6 +42,7 @@ GEM
       minitest (>= 5.1)
       mutex_m
       tzinfo (~> 2.0)
+    ast (2.4.2)
     base64 (0.2.0)
     bcrypt_pbkdf (1.1.0)
     bigdecimal (3.1.5)
@@ -63,6 +64,8 @@ GEM
     irb (1.11.0)
       rdoc
       reline (>= 0.3.8)
+    json (2.7.1)
+    language_server-protocol (3.17.0.3)
     loofah (2.22.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
@@ -72,6 +75,8 @@ GEM
     mutex_m (0.2.0)
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
+    net-sftp (4.0.0)
+      net-ssh (>= 5.0.0, < 8.0.0)
     net-ssh (7.2.1)
     nokogiri (1.16.0-arm64-darwin)
       racc (~> 1.4)
@@ -79,6 +84,10 @@ GEM
       racc (~> 1.4)
     nokogiri (1.16.0-x86_64-linux)
       racc (~> 1.4)
+    parallel (1.24.0)
+    parser (3.3.0.5)
+      ast (~> 2.4.1)
+      racc
     psych (5.1.2)
       stringio
     racc (1.7.3)
@@ -105,20 +114,56 @@ GEM
       rake (>= 12.2)
       thor (~> 1.0, >= 1.2.2)
       zeitwerk (~> 2.6)
+    rainbow (3.1.1)
     rake (13.1.0)
     rdoc (6.6.2)
       psych (>= 4.0.0)
+    regexp_parser (2.9.0)
     reline (0.4.2)
       io-console (~> 0.5)
+    rexml (3.2.6)
+    rubocop (1.62.1)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.31.2)
+      parser (>= 3.3.0.4)
+    rubocop-minitest (0.35.0)
+      rubocop (>= 1.61, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-performance (1.20.2)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
+    rubocop-rails (2.24.0)
+      activesupport (>= 4.2.0)
+      rack (>= 1.1)
+      rubocop (>= 1.33.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-rails-omakase (1.0.0)
+      rubocop
+      rubocop-minitest
+      rubocop-performance
+      rubocop-rails
+    ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
-    sshkit (1.21.7)
+    sshkit (1.22.2)
+      base64
       mutex_m
       net-scp (>= 1.1.2)
+      net-sftp (>= 2.1.2)
       net-ssh (>= 2.8.0)
     stringio (3.1.0)
     thor (1.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
+    unicode-display_width (2.5.0)
     webrick (1.8.1)
     zeitwerk (2.6.12)
 
@@ -132,6 +177,7 @@ DEPENDENCIES
   kamal!
   mocha
   railties
+  rubocop-rails-omakase
 
 BUNDLED WITH
    2.4.3

kamal.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.executables = %w[ kamal ]
 
   spec.add_dependency "activesupport", ">= 7.0"
-  spec.add_dependency "sshkit", "~> 1.21"
+  spec.add_dependency "sshkit", ">= 1.22.2", "< 2.0"
   spec.add_dependency "net-ssh", "~> 7.0"
   spec.add_dependency "thor", "~> 1.2"
   spec.add_dependency "dotenv", "~> 2.8"

lib/kamal.rb

@@ -5,6 +5,6 @@ require "active_support"
 require "zeitwerk"
 
 loader = Zeitwerk::Loader.for_gem
-loader.ignore("#{__dir__}/kamal/sshkit_with_ext.rb")
+loader.ignore(File.join(__dir__, "kamal", "sshkit_with_ext.rb"))
 loader.setup
 loader.eager_load # We need all commands loaded.

lib/kamal/cli/accessory.rb

@@ -1,7 +1,7 @@
 class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "boot [NAME]", "Boot new accessory service on host (use NAME=all to boot all accessories)"
   def boot(name, login: true)
-    mutating do
+    with_lock do
       if name == "all"
         KAMAL.accessory_names.each { |accessory_name| boot(accessory_name) }
       else
@@ -21,7 +21,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "upload [NAME]", "Upload accessory files to host", hide: true
   def upload(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           accessory.files.each do |(local, remote)|
@@ -38,7 +38,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "directories [NAME]", "Create accessory directories on host", hide: true
   def directories(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           accessory.directories.keys.each do |host_path|
@@ -51,7 +51,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container; use NAME=all to boot all accessories)"
   def reboot(name)
-    mutating do
+    with_lock do
       if name == "all"
         KAMAL.accessory_names.each { |accessory_name| reboot(accessory_name) }
       else
@@ -70,7 +70,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "start [NAME]", "Start existing accessory container on host"
   def start(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *KAMAL.auditor.record("Started #{name} accessory"), verbosity: :debug
@@ -82,7 +82,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "stop [NAME]", "Stop existing accessory container on host"
   def stop(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *KAMAL.auditor.record("Stopped #{name} accessory"), verbosity: :debug
@@ -94,7 +94,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "restart [NAME]", "Restart existing accessory container on host"
   def restart(name)
-    mutating do
+    with_lock do
       with_accessory(name) do
         stop(name)
         start(name)
@@ -107,8 +107,9 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     if name == "all"
       KAMAL.accessory_names.each { |accessory_name| details(accessory_name) }
     else
+      type = "Accessory #{name}"
       with_accessory(name) do |accessory, hosts|
-        on(hosts) { puts capture_with_info(*accessory.info) }
+        on(hosts) { puts_by_host host, capture_with_info(*accessory.info), type: type }
       end
     end
   end
@@ -173,17 +174,12 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "remove [NAME]", "Remove accessory container, image and data directory from host (use NAME=all to remove all accessories)"
   option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def remove(name)
-    mutating do
+    confirming "This will remove all containers, images and data directories for #{name}. Are you sure?" do
-      if name == "all"
+      with_lock do
-        KAMAL.accessory_names.each { |accessory_name| remove(accessory_name) }
+        if name == "all"
-      else
+          KAMAL.accessory_names.each { |accessory_name| remove_accessory(accessory_name) }
-        if options[:confirmed] || ask("This will remove all containers, images and data directories for #{name}. Are you sure?", limited_to: %w( y N ), default: "N") == "y"
+        else
-          with_accessory(name) do
+          remove_accessory(name)
-            stop(name)
-            remove_container(name)
-            remove_image(name)
-            remove_service_directory(name)
-          end
         end
       end
     end
@@ -191,7 +187,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "remove_container [NAME]", "Remove accessory container from host", hide: true
   def remove_container(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *KAMAL.auditor.record("Remove #{name} accessory container"), verbosity: :debug
@@ -203,7 +199,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "remove_image [NAME]", "Remove accessory image from host", hide: true
   def remove_image(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *KAMAL.auditor.record("Removed #{name} accessory image"), verbosity: :debug
@@ -215,7 +211,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "remove_service_directory [NAME]", "Remove accessory directory used for uploaded files and data directories from host", hide: true
   def remove_service_directory(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *accessory.remove_service_directory
@@ -226,7 +222,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   private
     def with_accessory(name)
-      if accessory = KAMAL.accessory(name)
+      if KAMAL.config.accessory(name)
+        accessory = KAMAL.accessory(name)
         yield accessory, accessory_hosts(accessory)
       else
         error_on_missing_accessory(name)
@@ -248,4 +245,13 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
         accessory.hosts
       end
     end
+
+    def remove_accessory(name)
+      with_accessory(name) do
+        stop(name)
+        remove_container(name)
+        remove_image(name)
+        remove_service_directory(name)
+      end
+    end
 end

lib/kamal/cli/app.rb

@@ -1,78 +1,45 @@
 class Kamal::Cli::App < Kamal::Cli::Base
   desc "boot", "Boot app on servers (or reboot app if already running)"
   def boot
-    mutating do
+    with_lock do
-      hold_lock_on_error do
+      say "Get most recent version available as an image...", :magenta unless options[:version]
-        say "Get most recent version available as an image...", :magenta unless options[:version]
+      using_version(version_or_latest) do |version|
-        using_version(version_or_latest) do |version|
+        say "Start container with version #{version} using a #{KAMAL.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
-          say "Start container with version #{version} using a #{KAMAL.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
 
-          on(KAMAL.hosts) do
+        # Assets are prepared in a separate step to ensure they are on all hosts before booting
-            execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
+        on(KAMAL.hosts) do
-            execute *KAMAL.app.tag_current_image_as_latest
+          KAMAL.roles_on(host).each do |role|
-
+            Kamal::Cli::App::PrepareAssets.new(host, role, self).run
-            KAMAL.roles_on(host).each do |role|
-              app = KAMAL.app(role: role)
-
-              if role.assets?
-                execute *app.extract_assets
-                old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-                execute *app.sync_asset_volumes(old_version: old_version)
-              end
-            end
           end
+        end
 
-          on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
+        # Primary hosts and roles are returned first, so they can open the barrier
-            KAMAL.roles_on(host).each do |role|
+        barrier = Kamal::Cli::Healthcheck::Barrier.new if KAMAL.roles.many?
-              app = KAMAL.app(role: role)
-              auditor = KAMAL.auditor(role: role)
 
-              if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
+        on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
-                tmp_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
+          KAMAL.roles_on(host).each do |role|
-                info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
+            Kamal::Cli::App::Boot.new(host, role, self, version, barrier).run
-                execute *auditor.record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
-                execute *app.rename_container(version: version, new_version: tmp_version)
-              end
-
-              old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-
-              execute *app.tie_cord(role.cord_host_file) if role.uses_cord?
-
-              execute *auditor.record("Booted app version #{version}"), verbosity: :debug
-
-              execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
-
-              Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
-
-              if old_version.present?
-                if role.uses_cord?
-                  cord = capture_with_info(*app.cord(version: old_version), raise_on_non_zero_exit: false).strip
-                  if cord.present?
-                    execute *app.cut_cord(cord)
-                    Kamal::Cli::Healthcheck::Poller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
-                  end
-                end
-
-                execute *app.stop(version: old_version), raise_on_non_zero_exit: false
-
-                execute *app.clean_up_assets if role.assets?
-              end
-            end
           end
         end
+
+        # Tag once the app booted on all hosts
+        on(KAMAL.hosts) do |host|
+          execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
+          execute *KAMAL.app.tag_latest_image
+        end
       end
     end
   end
 
   desc "start", "Start existing app container on servers"
   def start
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
           execute *KAMAL.auditor.record("Started app version #{KAMAL.config.version}"), verbosity: :debug
-          execute *KAMAL.app(role: role).start, raise_on_non_zero_exit: false
+          execute *KAMAL.app(role: role, host: host).start, raise_on_non_zero_exit: false
         end
       end
     end
@@ -80,13 +47,13 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "stop", "Stop app container on servers"
   def stop
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
           execute *KAMAL.auditor.record("Stopped app", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).stop, raise_on_non_zero_exit: false
+          execute *KAMAL.app(role: role, host: host).stop, raise_on_non_zero_exit: false
         end
       end
     end
@@ -99,21 +66,23 @@ class Kamal::Cli::App < Kamal::Cli::Base
       roles = KAMAL.roles_on(host)
 
       roles.each do |role|
-        puts_by_host host, capture_with_info(*KAMAL.app(role: role).info)
+        puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).info)
       end
     end
   end
 
-  desc "exec [CMD]", "Execute a custom command on servers (use --help to show options)"
+  desc "exec [CMD]", "Execute a custom command on servers within the app container (use --help to show options)"
   option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
   option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
+  option :env, aliases: "-e", type: :hash, desc: "Set environment variables for the command"
   def exec(cmd)
+    env = options[:env]
     case
     when options[:interactive] && options[:reuse]
       say "Get current version of running container...", :magenta unless options[:version]
       using_version(options[:version] || current_running_version) do |version|
         say "Launching interactive command with version #{version} via SSH from existing container on #{KAMAL.primary_host}...", :magenta
-        run_locally { exec KAMAL.app(role: KAMAL.primary_role).execute_in_existing_container_over_ssh(cmd, host: KAMAL.primary_host) }
+        run_locally { exec KAMAL.app(role: KAMAL.primary_role, host: KAMAL.primary_host).execute_in_existing_container_over_ssh(cmd, env: env) }
       end
 
     when options[:interactive]
@@ -121,7 +90,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
       using_version(version_or_latest) do |version|
         say "Launching interactive command with version #{version} via SSH from new container on #{KAMAL.primary_host}...", :magenta
         run_locally do
-          exec KAMAL.app(role: KAMAL.primary_role).execute_in_new_container_over_ssh(cmd, host: KAMAL.primary_host)
+          exec KAMAL.app(role: KAMAL.primary_role, host: KAMAL.primary_host).execute_in_new_container_over_ssh(cmd, env: env)
         end
       end
 
@@ -135,7 +104,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
           roles.each do |role|
             execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}", role: role), verbosity: :debug
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_existing_container(cmd))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).execute_in_existing_container(cmd, env: env))
           end
         end
       end
@@ -149,7 +118,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
           roles.each do |role|
             execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_new_container(cmd))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).execute_in_new_container(cmd, env: env))
           end
         end
       end
@@ -164,19 +133,21 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "stale_containers", "Detect app stale containers"
   option :stop, aliases: "-s", type: :boolean, default: false, desc: "Stop the stale containers found"
   def stale_containers
-    mutating do
+    stop = options[:stop]
-      stop = options[:stop]
-
-      cli = self
 
+    with_lock_if_stopping do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
-          cli.send(:stale_versions, host: host, role: role).each do |version|
+          app = KAMAL.app(role: role, host: host)
+          versions = capture_with_info(*app.list_versions, raise_on_non_zero_exit: false).split("\n")
+          versions -= [ capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip ]
+
+          versions.each do |version|
             if stop
               puts_by_host host, "Stopping stale container for role #{role} with version #{version}"
-              execute *KAMAL.app(role: role).stop(version: version), raise_on_non_zero_exit: false
+              execute *app.stop(version: version), raise_on_non_zero_exit: false
             else
               puts_by_host host,  "Detected stale container for role #{role} with version #{version} (use `kamal app stale_containers --stop` to stop)"
             end
@@ -207,11 +178,12 @@ class Kamal::Cli::App < Kamal::Cli::Base
       run_locally do
         info "Following logs on #{KAMAL.primary_host}..."
 
-        KAMAL.specific_roles ||= ["web"]
+        KAMAL.specific_roles ||= [ "web" ]
         role = KAMAL.roles_on(KAMAL.primary_host).first
 
-        info KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
+        app = KAMAL.app(role: role, host: host)
-        exec KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
+        info app.follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
+        exec app.follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
       end
     else
       lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
@@ -221,7 +193,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
         roles.each do |role|
           begin
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).logs(since: since, lines: lines, grep: grep))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).logs(since: since, lines: lines, grep: grep))
           rescue SSHKit::Command::Failed
             puts_by_host host, "Nothing found"
           end
@@ -232,7 +204,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove", "Remove app containers and images from servers"
   def remove
-    mutating do
+    with_lock do
       stop
       remove_containers
       remove_images
@@ -241,13 +213,13 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true
   def remove_container(version)
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
           execute *KAMAL.auditor.record("Removed app container with version #{version}", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).remove_container(version: version)
+          execute *KAMAL.app(role: role, host: host).remove_container(version: version)
         end
       end
     end
@@ -255,13 +227,13 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove_containers", "Remove all app containers from servers", hide: true
   def remove_containers
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
           execute *KAMAL.auditor.record("Removed all app containers", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).remove_containers
+          execute *KAMAL.app(role: role, host: host).remove_containers
         end
       end
     end
@@ -269,7 +241,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove_images", "Remove all app images from servers", hide: true
   def remove_images
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Removed all app images"), verbosity: :debug
         execute *KAMAL.app.remove_images
@@ -281,7 +253,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
   def version
     on(KAMAL.hosts) do |host|
       role = KAMAL.roles_on(host).first
-      puts_by_host host, capture_with_info(*KAMAL.app(role: role).current_running_version).strip
+      puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).current_running_version).strip
     end
   end
 
@@ -304,23 +276,20 @@ class Kamal::Cli::App < Kamal::Cli::Base
       version = nil
       on(host) do
         role = KAMAL.roles_on(host).first
-        version = capture_with_info(*KAMAL.app(role: role).current_running_version).strip
+        version = capture_with_info(*KAMAL.app(role: role, host: host).current_running_version).strip
       end
       version.presence
     end
 
-    def stale_versions(host:, role:)
+    def version_or_latest
-      versions = nil
+      options[:version] || KAMAL.config.latest_tag
-      on(host) do
-        versions = \
-          capture_with_info(*KAMAL.app(role: role).list_versions, raise_on_non_zero_exit: false)
-          .split("\n")
-          .drop(1)
-      end
-      versions
     end
 
-    def version_or_latest
+    def with_lock_if_stopping
-      options[:version] || "latest"
+      if options[:stop]
+        with_lock { yield }
+      else
+        yield
+      end
     end
 end

lib/kamal/cli/app/boot.rb

@@ -0,0 +1,119 @@
+class Kamal::Cli::App::Boot
+  attr_reader :host, :role, :version, :barrier, :sshkit
+  delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, to: :sshkit
+  delegate :uses_cord?, :assets?, :running_traefik?, to: :role
+
+  def initialize(host, role, sshkit, version, barrier)
+    @host = host
+    @role = role
+    @version = version
+    @barrier = barrier
+    @sshkit = sshkit
+  end
+
+  def run
+    old_version = old_version_renamed_if_clashing
+
+    wait_at_barrier if queuer?
+
+    begin
+      start_new_version
+    rescue => e
+      close_barrier if gatekeeper?
+      stop_new_version
+      raise
+    end
+
+    release_barrier if gatekeeper?
+
+    if old_version
+      stop_old_version(old_version)
+    end
+  end
+
+  private
+    def old_version_renamed_if_clashing
+      if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
+        renamed_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
+        info "Renaming container #{version} to #{renamed_version} as already deployed on #{host}"
+        audit("Renaming container #{version} to #{renamed_version}")
+        execute *app.rename_container(version: version, new_version: renamed_version)
+      end
+
+      capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip.presence
+    end
+
+    def start_new_version
+      audit "Booted app version #{version}"
+
+      execute *app.tie_cord(role.cord_host_file) if uses_cord?
+      hostname = "#{host.to_s[0...51].gsub(/\.+$/, '')}-#{SecureRandom.hex(6)}"
+      execute *app.run(hostname: hostname)
+      Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+    end
+
+    def stop_new_version
+      execute *app.stop(version: version), raise_on_non_zero_exit: false
+    end
+
+    def stop_old_version(version)
+      if uses_cord?
+        cord = capture_with_info(*app.cord(version: version), raise_on_non_zero_exit: false).strip
+        if cord.present?
+          execute *app.cut_cord(cord)
+          Kamal::Cli::Healthcheck::Poller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+        end
+      end
+
+      execute *app.stop(version: version), raise_on_non_zero_exit: false
+
+      execute *app.clean_up_assets if assets?
+    end
+
+    def release_barrier
+      if barrier.open
+        info "First #{KAMAL.primary_role} container is healthy on #{host}, booting other roles"
+      end
+    end
+
+    def wait_at_barrier
+      info "Waiting for the first healthy #{KAMAL.primary_role} container before booting #{role} on #{host}..."
+      barrier.wait
+      info "First #{KAMAL.primary_role} container is healthy, booting #{role} on #{host}..."
+    rescue Kamal::Cli::Healthcheck::Error
+      info "First #{KAMAL.primary_role} container is unhealthy, not booting #{role} on #{host}"
+      raise
+    end
+
+    def close_barrier
+      if barrier.close
+        info "First #{KAMAL.primary_role} container is unhealthy on #{host}, not booting other roles"
+        error capture_with_info(*app.logs(version: version))
+        error capture_with_info(*app.container_health_log(version: version))
+      end
+    end
+
+    def barrier_role?
+      role == KAMAL.primary_role
+    end
+
+    def app
+      @app ||= KAMAL.app(role: role, host: host)
+    end
+
+    def auditor
+      @auditor = KAMAL.auditor(role: role)
+    end
+
+    def audit(message)
+      execute *auditor.record(message), verbosity: :debug
+    end
+
+    def gatekeeper?
+      barrier && barrier_role?
+    end
+
+    def queuer?
+      barrier && !barrier_role?
+    end
+end

lib/kamal/cli/app/prepare_assets.rb

@@ -0,0 +1,24 @@
+class Kamal::Cli::App::PrepareAssets
+  attr_reader :host, :role, :sshkit
+  delegate :execute, :capture_with_info, :info, to: :sshkit
+  delegate :assets?, to: :role
+
+  def initialize(host, role, sshkit)
+    @host = host
+    @role = role
+    @sshkit = sshkit
+  end
+
+  def run
+    if assets?
+      execute *app.extract_assets
+      old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+      execute *app.sync_asset_volumes(old_version: old_version)
+    end
+  end
+
+  private
+    def app
+      @app ||= KAMAL.app(role: role, host: host)
+    end
+end

lib/kamal/cli/base.rb

@@ -73,34 +73,43 @@ module Kamal::Cli
       def print_runtime
         started_at = Time.now
         yield
-        return Time.now - started_at
+        Time.now - started_at
       ensure
         runtime = Time.now - started_at
         puts "  Finished all in #{sprintf("%.1f seconds", runtime)}"
       end
 
-      def mutating
+      def with_lock
-        return yield if KAMAL.holding_lock?
+        if KAMAL.holding_lock?
-
-        run_hook "pre-connect"
-
-        ensure_run_directory
-
-        acquire_lock
-
-        begin
           yield
-        rescue
+        else
-          if KAMAL.hold_lock_on_error?
+          ensure_run_and_locks_directory
-            error "  \e[31mDeploy lock was not released\e[0m"
+
-          else
+          acquire_lock
-            release_lock
+
+          begin
+            yield
+          rescue
+            begin
+              release_lock
+            rescue => e
+              say "Error releasing the deploy lock: #{e.message}", :red
+            end
+            raise
           end
 
-          raise
+          release_lock
         end
+      end
 
-        release_lock
+      def confirming(question)
+        return yield if options[:confirmed]
+
+        if ask(question, limited_to: %w[ y N ], default: "N") == "y"
+          yield
+        else
+          say "Aborted", :red
+        end
       end
 
       def acquire_lock
@@ -131,29 +140,28 @@ module Kamal::Cli
         end
       end
 
-      def hold_lock_on_error
-        if KAMAL.hold_lock_on_error?
-          yield
-        else
-          KAMAL.hold_lock_on_error = true
-          yield
-          KAMAL.hold_lock_on_error = false
-        end
-      end
-
       def run_hook(hook, **extra_details)
         if !options[:skip_hooks] && KAMAL.hook.hook_exists?(hook)
           details = { hosts: KAMAL.hosts.join(","), command: command, subcommand: subcommand }
 
           say "Running the #{hook} hook...", :magenta
           run_locally do
-            KAMAL.with_verbosity(:debug) { execute *KAMAL.hook.run(hook, **details, **extra_details) }
+            execute *KAMAL.hook.run(hook, **details, **extra_details)
-          rescue SSHKit::Command::Failed
+          rescue SSHKit::Command::Failed => e
-            raise HookError.new("Hook `#{hook}` failed")
+            raise HookError.new("Hook `#{hook}` failed:\n#{e.message}")
           end
         end
       end
 
+      def on(*args, &block)
+        if !KAMAL.connected?
+          run_hook "pre-connect"
+          KAMAL.connected = true
+        end
+
+        super
+      end
+
       def command
         @kamal_command ||= begin
           invocation_class, invocation_commands = *first_invocation
@@ -176,10 +184,14 @@ module Kamal::Cli
         instance_variable_get("@_invocations").first
       end
 
-      def ensure_run_directory
+      def ensure_run_and_locks_directory
         on(KAMAL.hosts) do
           execute(*KAMAL.server.ensure_run_directory)
         end
+
+        on(KAMAL.primary_host) do
+          execute(*KAMAL.lock.ensure_locks_directory)
+        end
       end
-    end
+  end
 end

lib/kamal/cli/build.rb

@@ -5,39 +5,50 @@ class Kamal::Cli::Build < Kamal::Cli::Base
 
   desc "deliver", "Build app and push app image to registry then pull image on servers"
   def deliver
-    mutating do
+    push
-      push
+    pull
-      pull
-    end
   end
 
   desc "push", "Build and push app image to registry"
   def push
-    mutating do
+    cli = self
-      cli = self
 
-      verify_local_dependencies
+    verify_local_dependencies
-      run_hook "pre-build"
+    run_hook "pre-build"
 
-      if (uncommitted_changes = Kamal::Git.uncommitted_changes).present?
+    uncommitted_changes = Kamal::Git.uncommitted_changes
-        say "The following paths have uncommitted changes:\n #{uncommitted_changes}", :yellow
+
+    if KAMAL.config.builder.git_clone?
+      if uncommitted_changes.present?
+        say "Building from a local git clone, so ignoring these uncommitted changes:\n #{uncommitted_changes}", :yellow
       end
 
       run_locally do
-        begin
+        Clone.new(self).prepare
-          KAMAL.with_verbosity(:debug) do
+      end
-            execute *KAMAL.builder.push
+    elsif uncommitted_changes.present?
-          end
+      say "Building with uncommitted changes:\n #{uncommitted_changes}", :yellow
-        rescue SSHKit::Command::Failed => e
+    end
-          if e.message =~ /(no builder)|(no such file or directory)/
-            error "Missing compatible builder, so creating a new one first"
 
-            if cli.create
+    # Get the command here to ensure the Dir.chdir doesn't interfere with it
-              KAMAL.with_verbosity(:debug) { execute *KAMAL.builder.push }
+    push = KAMAL.builder.push
+
+    run_locally do
+      begin
+        KAMAL.with_verbosity(:debug) do
+          Dir.chdir(KAMAL.config.builder.build_directory) { execute *push }
+        end
+      rescue SSHKit::Command::Failed => e
+        if e.message =~ /(no builder)|(no such file or directory)/
+          warn "Missing compatible builder, so creating a new one first"
+
+          if cli.create
+            KAMAL.with_verbosity(:debug) do
+              Dir.chdir(KAMAL.config.builder.build_directory) { execute *push }
             end
-          else
-            raise
           end
+        else
+          raise
         end
       end
     end
@@ -45,34 +56,30 @@ class Kamal::Cli::Build < Kamal::Cli::Base
 
   desc "pull", "Pull app image from registry onto servers"
   def pull
-    mutating do
+    on(KAMAL.hosts) do
-      on(KAMAL.hosts) do
+      execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
-        execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
+      execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
-        execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
+      execute *KAMAL.builder.pull
-        execute *KAMAL.builder.pull
+      execute *KAMAL.builder.validate_image
-        execute *KAMAL.builder.validate_image
-      end
     end
   end
 
   desc "create", "Create a build setup"
   def create
-    mutating do
+    if (remote_host = KAMAL.config.builder.remote_host)
-      if (remote_host = KAMAL.config.builder.remote_host)
+      connect_to_remote_host(remote_host)
-        connect_to_remote_host(remote_host)
+    end
-      end
 
-      run_locally do
+    run_locally do
-        begin
+      begin
-          debug "Using builder: #{KAMAL.builder.name}"
+        debug "Using builder: #{KAMAL.builder.name}"
-          execute *KAMAL.builder.create
+        execute *KAMAL.builder.create
-        rescue SSHKit::Command::Failed => e
+      rescue SSHKit::Command::Failed => e
-          if e.message =~ /stderr=(.*)/
+        if e.message =~ /stderr=(.*)/
-            error "Couldn't create remote builder: #{$1}"
+          error "Couldn't create remote builder: #{$1}"
-            false
+          false
-          else
+        else
-            raise
+          raise
-          end
         end
       end
     end
@@ -80,11 +87,9 @@ class Kamal::Cli::Build < Kamal::Cli::Base
 
   desc "remove", "Remove build setup"
   def remove
-    mutating do
+    run_locally do
-      run_locally do
+      debug "Using builder: #{KAMAL.builder.name}"
-        debug "Using builder: #{KAMAL.builder.name}"
+      execute *KAMAL.builder.remove
-        execute *KAMAL.builder.remove
-      end
     end
   end
 
@@ -114,8 +119,11 @@ class Kamal::Cli::Build < Kamal::Cli::Base
     def connect_to_remote_host(remote_host)
       remote_uri = URI.parse(remote_host)
       if remote_uri.scheme == "ssh"
-        options = { user: remote_uri.user, port: remote_uri.port }.compact
+        host = SSHKit::Host.new(
-        on(remote_uri.host, options) do
+          hostname: remote_uri.host,
+          ssh_options: { user: remote_uri.user, port: remote_uri.port }.compact
+        )
+        on(host, options) do
           execute "true"
         end
       end

lib/kamal/cli/build/clone.rb

@@ -0,0 +1,61 @@
+require "uri"
+
+class Kamal::Cli::Build::Clone
+  attr_reader :sshkit
+  delegate :info, :error, :execute, :capture_with_info, to: :sshkit
+
+  def initialize(sshkit)
+    @sshkit = sshkit
+  end
+
+  def prepare
+    begin
+      clone_repo
+    rescue SSHKit::Command::Failed => e
+      if e.message =~ /already exists and is not an empty directory/
+        reset
+      else
+        raise Kamal::Cli::Build::BuildError, "Failed to clone repo: #{e.message}"
+      end
+    end
+
+    validate!
+  rescue Kamal::Cli::Build::BuildError => e
+    error "Error preparing clone: #{e.message}, deleting and retrying..."
+
+    FileUtils.rm_rf KAMAL.config.builder.clone_directory
+    clone_repo
+    validate!
+  end
+
+  private
+    def clone_repo
+      info "Cloning repo into build directory `#{KAMAL.config.builder.build_directory}`..."
+
+      FileUtils.mkdir_p KAMAL.config.builder.clone_directory
+      execute *KAMAL.builder.clone
+    end
+
+    def reset
+      info "Resetting local clone as `#{KAMAL.config.builder.build_directory}` already exists..."
+
+      KAMAL.builder.clone_reset_steps.each { |step| execute *step }
+    rescue SSHKit::Command::Failed => e
+      raise Kamal::Cli::Build::BuildError, "Failed to clone repo: #{e.message}"
+    end
+
+    def validate!
+      status = capture_with_info(*KAMAL.builder.clone_status).strip
+
+      unless status.empty?
+        raise Kamal::Cli::Build::BuildError, "Clone in #{KAMAL.config.builder.build_directory} is dirty, #{status}"
+      end
+
+      revision = capture_with_info(*KAMAL.builder.clone_revision).strip
+      if revision != Kamal::Git.revision
+        raise Kamal::Cli::Build::BuildError, "Clone in #{KAMAL.config.builder.build_directory} is not on the correct revision, expected `#{Kamal::Git.revision}` but got `#{revision}`"
+      end
+    rescue SSHKit::Command::Failed => e
+      raise Kamal::Cli::Build::BuildError, "Failed to validate clone: #{e.message}"
+    end
+end

lib/kamal/cli/env.rb

@@ -3,26 +3,26 @@ require "tempfile"
 class Kamal::Cli::Env < Kamal::Cli::Base
   desc "push", "Push the env file to the remote hosts"
   def push
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pushed env files"), verbosity: :debug
 
         KAMAL.roles_on(host).each do |role|
-          execute *KAMAL.app(role: role).make_env_directory
+          execute *KAMAL.app(role: role, host: host).make_env_directory
-          upload! StringIO.new(role.env_file), role.host_env_file_path, mode: 400
+          upload! role.env(host).secrets_io, role.env(host).secrets_file, mode: 400
         end
       end
 
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.traefik.make_env_directory
-        upload! StringIO.new(KAMAL.traefik.env_file), KAMAL.traefik.host_env_file_path, mode: 400
+        upload! KAMAL.traefik.env.secrets_io, KAMAL.traefik.env.secrets_file, mode: 400
       end
 
       on(KAMAL.accessory_hosts) do
         KAMAL.accessories_on(host).each do |accessory|
           accessory_config = KAMAL.config.accessory(accessory)
           execute *KAMAL.accessory(accessory).make_env_directory
-          upload! StringIO.new(accessory_config.env_file), accessory_config.host_env_file_path, mode: 400
+          upload! accessory_config.env.secrets_io, accessory_config.env.secrets_file, mode: 400
         end
       end
     end
@@ -30,12 +30,12 @@ class Kamal::Cli::Env < Kamal::Cli::Base
 
   desc "delete", "Delete the env file from the remote hosts"
   def delete
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Deleted env files"), verbosity: :debug
 
         KAMAL.roles_on(host).each do |role|
-          execute *KAMAL.app(role: role).remove_env_file
+          execute *KAMAL.app(role: role, host: host).remove_env_file
         end
       end
 

lib/kamal/cli/healthcheck.rb

@@ -1,21 +0,0 @@
-class Kamal::Cli::Healthcheck < Kamal::Cli::Base
-  default_command :perform
-
-  desc "perform", "Health check current app version"
-  def perform
-    raise "The primary host is not configured to run Traefik" unless KAMAL.config.role(KAMAL.config.primary_role).running_traefik?
-    on(KAMAL.primary_host) do
-      begin
-        execute *KAMAL.healthcheck.run
-        Poller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
-      rescue Poller::HealthcheckError => e
-        error capture_with_info(*KAMAL.healthcheck.logs)
-        error capture_with_pretty_json(*KAMAL.healthcheck.container_health_log)
-        raise
-      ensure
-        execute *KAMAL.healthcheck.stop, raise_on_non_zero_exit: false
-        execute *KAMAL.healthcheck.remove, raise_on_non_zero_exit: false
-      end
-    end
-  end
-end

lib/kamal/cli/healthcheck/barrier.rb

@@ -0,0 +1,31 @@
+class Kamal::Cli::Healthcheck::Barrier
+  def initialize
+    @ivar = Concurrent::IVar.new
+  end
+
+  def close
+    set(false)
+  end
+
+  def open
+    set(true)
+  end
+
+  def wait
+    unless opened?
+      raise Kamal::Cli::Healthcheck::Error.new("Halted at barrier")
+    end
+  end
+
+  private
+    def opened?
+      @ivar.value
+    end
+
+    def set(value)
+      @ivar.set(value)
+      true
+    rescue Concurrent::MultipleAssignmentError
+      false
+    end
+end

lib/kamal/cli/healthcheck/error.rb

@@ -0,0 +1,2 @@
+class Kamal::Cli::Healthcheck::Error < StandardError
+end

lib/kamal/cli/healthcheck/poller.rb

@@ -3,7 +3,6 @@ module Kamal::Cli::Healthcheck::Poller
 
   TRAEFIK_UPDATE_DELAY = 5
 
-  class HealthcheckError < StandardError; end
 
   def wait_for_healthy(pause_after_ready: false, &block)
     attempt = 1
@@ -16,9 +15,9 @@ module Kamal::Cli::Healthcheck::Poller
       when "running" # No health check configured
         sleep KAMAL.config.readiness_delay if pause_after_ready
       else
-        raise HealthcheckError, "container not ready (#{status})"
+        raise Kamal::Cli::Healthcheck::Error, "container not ready (#{status})"
       end
-    rescue HealthcheckError => e
+    rescue Kamal::Cli::Healthcheck::Error => e
       if attempt <= max_attempts
         info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
         sleep attempt
@@ -41,9 +40,9 @@ module Kamal::Cli::Healthcheck::Poller
       when "unhealthy"
         sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
       else
-        raise HealthcheckError, "container not unhealthy (#{status})"
+        raise Kamal::Cli::Healthcheck::Error, "container not unhealthy (#{status})"
       end
-    rescue HealthcheckError => e
+    rescue Kamal::Cli::Healthcheck::Error => e
       if attempt <= max_attempts
         info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
         sleep attempt

lib/kamal/cli/main.rb

@@ -3,14 +3,14 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def setup
     print_runtime do
-      mutating do
+      with_lock do
         invoke_options = deploy_options
 
         say "Ensure Docker is installed...", :magenta
         invoke "kamal:cli:server:bootstrap", [], invoke_options
 
-        say "Push env files...", :magenta
+        say "Evaluate and push env files...", :magenta
-        invoke "kamal:cli:env:push", [], invoke_options
+        invoke "kamal:cli:main:envify", [], invoke_options
 
         invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options
         deploy
@@ -22,30 +22,25 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def deploy
     runtime = print_runtime do
-      mutating do
+      invoke_options = deploy_options
-        invoke_options = deploy_options
 
-        say "Log into image registry...", :magenta
+      say "Log into image registry...", :magenta
-        invoke "kamal:cli:registry:login", [], invoke_options
+      invoke "kamal:cli:registry:login", [], invoke_options
 
-        if options[:skip_push]
+      if options[:skip_push]
-          say "Pull app image...", :magenta
+        say "Pull app image...", :magenta
-          invoke "kamal:cli:build:pull", [], invoke_options
+        invoke "kamal:cli:build:pull", [], invoke_options
-        else
+      else
-          say "Build and push app image...", :magenta
+        say "Build and push app image...", :magenta
-          invoke "kamal:cli:build:deliver", [], invoke_options
+        invoke "kamal:cli:build:deliver", [], invoke_options
-        end
+      end
 
+      with_lock do
         run_hook "pre-deploy"
 
         say "Ensure Traefik is running...", :magenta
         invoke "kamal:cli:traefik:boot", [], invoke_options
 
-        if KAMAL.config.role(KAMAL.config.primary_role).running_traefik?
-          say "Ensure app can pass healthcheck...", :magenta
-          invoke "kamal:cli:healthcheck:perform", [], invoke_options
-        end
-
         say "Detect stale containers...", :magenta
         invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
 
@@ -63,22 +58,19 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def redeploy
     runtime = print_runtime do
-      mutating do
+      invoke_options = deploy_options
-        invoke_options = deploy_options
 
-        if options[:skip_push]
+      if options[:skip_push]
-          say "Pull app image...", :magenta
+        say "Pull app image...", :magenta
-          invoke "kamal:cli:build:pull", [], invoke_options
+        invoke "kamal:cli:build:pull", [], invoke_options
-        else
+      else
-          say "Build and push app image...", :magenta
+        say "Build and push app image...", :magenta
-          invoke "kamal:cli:build:deliver", [], invoke_options
+        invoke "kamal:cli:build:deliver", [], invoke_options
-        end
+      end
 
+      with_lock do
         run_hook "pre-deploy"
 
-        say "Ensure app can pass healthcheck...", :magenta
-        invoke "kamal:cli:healthcheck:perform", [], invoke_options
-
         say "Detect stale containers...", :magenta
         invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
 
@@ -93,7 +85,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   def rollback(version)
     rolled_back = false
     runtime = print_runtime do
-      mutating do
+      with_lock do
         invoke_options = deploy_options
 
         KAMAL.config.version = version
@@ -185,19 +177,23 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       env_path          = ".env"
     end
 
-    File.write(env_path, ERB.new(File.read(env_template_path), trim_mode: "-").result, perm: 0600)
+    if Pathname.new(File.expand_path(env_template_path)).exist?
+      File.write(env_path, ERB.new(File.read(env_template_path), trim_mode: "-").result, perm: 0600)
 
-    unless options[:skip_push]
+      unless options[:skip_push]
-      reload_envs
+        reload_envs
-      invoke "kamal:cli:env:push", options
+        invoke "kamal:cli:env:push", options
+      end
+    else
+      puts "Skipping envify (no #{env_template_path} exist)"
     end
   end
 
   desc "remove", "Remove Traefik, app, accessories, and registry session from servers"
   option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def remove
-    mutating do
+    confirming "This will remove all containers and images. Are you sure?" do
-      if options[:confirmed] || ask("This will remove all containers and images. Are you sure?", limited_to: %w( y N ), default: "N") == "y"
+      with_lock do
         invoke "kamal:cli:traefik:remove", [], options.without(:confirmed)
         invoke "kamal:cli:app:remove", [], options.without(:confirmed)
         invoke "kamal:cli:accessory:remove", [ "all" ], options
@@ -223,9 +219,6 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   desc "env", "Manage environment files"
   subcommand "env", Kamal::Cli::Env
 
-  desc "healthcheck", "Healthcheck application"
-  subcommand "healthcheck", Kamal::Cli::Healthcheck
-
   desc "lock", "Manage the deploy lock"
   subcommand "lock", Kamal::Cli::Lock
 
@@ -246,11 +239,11 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       begin
         on(KAMAL.hosts) do
           KAMAL.roles_on(host).each do |role|
-            container_id = capture_with_info(*KAMAL.app(role: role).container_id_for_version(version))
+            container_id = capture_with_info(*KAMAL.app(role: role, host: host).container_id_for_version(version))
             raise "Container not found" unless container_id.present?
           end
         end
-      rescue SSHKit::Runner::ExecuteError => e
+      rescue SSHKit::Runner::ExecuteError, SSHKit::Runner::MultipleExecuteError => e
         if e.message =~ /Container not found/
           say "Error looking for container version #{version}: #{e.message}"
           return false

lib/kamal/cli/prune.rb

@@ -1,7 +1,7 @@
 class Kamal::Cli::Prune < Kamal::Cli::Base
   desc "all", "Prune unused images and stopped containers"
   def all
-    mutating do
+    with_lock do
       containers
       images
     end
@@ -9,7 +9,7 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
 
   desc "images", "Prune unused images"
   def images
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pruned images"), verbosity: :debug
         execute *KAMAL.prune.dangling_images
@@ -24,7 +24,7 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
     retain = options.fetch(:retain, KAMAL.config.retain_containers)
     raise "retain must be at least 1" if retain < 1
 
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug
         execute *KAMAL.prune.app_containers(retain: retain)

lib/kamal/cli/server.rb

@@ -1,25 +1,49 @@
 class Kamal::Cli::Server < Kamal::Cli::Base
+  desc "exec", "Run a custom command on the server (use --help to show options)"
+  option :interactive, type: :boolean, aliases: "-i", default: false, desc: "Run the command interactively (use for console/bash)"
+  def exec(cmd)
+    hosts = KAMAL.hosts | KAMAL.accessory_hosts
+
+    case
+    when options[:interactive]
+      host = KAMAL.primary_host
+
+      say "Running '#{cmd}' on #{host} interactively...", :magenta
+
+      run_locally { exec KAMAL.server.run_over_ssh(cmd, host: host) }
+    else
+      say "Running '#{cmd}' on #{hosts.join(', ')}...", :magenta
+
+      on(hosts) do |host|
+        execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{host}"), verbosity: :debug
+        puts_by_host host, capture_with_info(cmd)
+      end
+    end
+  end
+
   desc "bootstrap", "Set up Docker to run Kamal apps"
   def bootstrap
-    missing = []
+    with_lock do
+      missing = []
 
-    on(KAMAL.hosts | KAMAL.accessory_hosts) do |host|
+      on(KAMAL.hosts | KAMAL.accessory_hosts) do |host|
-      unless execute(*KAMAL.docker.installed?, raise_on_non_zero_exit: false)
+        unless execute(*KAMAL.docker.installed?, raise_on_non_zero_exit: false)
-        if execute(*KAMAL.docker.superuser?, raise_on_non_zero_exit: false)
+          if execute(*KAMAL.docker.superuser?, raise_on_non_zero_exit: false)
-          info "Missing Docker on #{host}. Installing…"
+            info "Missing Docker on #{host}. Installing…"
-          execute *KAMAL.docker.install
+            execute *KAMAL.docker.install
-        else
+          else
-          missing << host
+            missing << host
+          end
         end
+
+        execute(*KAMAL.server.ensure_run_directory)
       end
 
-      execute(*KAMAL.server.ensure_run_directory)
+      if missing.any?
-    end
+        raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and either `wget` or `curl`. Install Docker manually: https://docs.docker.com/engine/install/"
+      end
 
-    if missing.any?
+      run_hook "docker-setup"
-      raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and either `wget` or `curl`. Install Docker manually: https://docs.docker.com/engine/install/"
     end
-
-    run_hook "docker-setup"
   end
 end

lib/kamal/cli/templates/deploy.yml

@@ -63,7 +63,7 @@ registry:
 #     directories:
 #       - data:/data
 
-# Configure custom arguments for Traefik
+# Configure custom arguments for Traefik. Be sure to reboot traefik when you modify it.
 # traefik:
 #   args:
 #     accesslog: true

lib/kamal/cli/traefik.rb

@@ -1,7 +1,7 @@
 class Kamal::Cli::Traefik < Kamal::Cli::Base
   desc "boot", "Boot Traefik on servers"
   def boot
-    mutating do
+    with_lock do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.registry.login
         execute *KAMAL.traefik.start_or_run
@@ -11,27 +11,30 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
 
   desc "reboot", "Reboot Traefik on servers (stop container, remove container, start new container)"
   option :rolling, type: :boolean, default: false, desc: "Reboot traefik on hosts in sequence, rather than in parallel"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def reboot
-    mutating do
+    confirming "This will cause a brief outage on each host. Are you sure?" do
-      host_groups = options[:rolling] ? KAMAL.traefik_hosts : [KAMAL.traefik_hosts]
+      with_lock do
-      host_groups.each do |hosts|
+        host_groups = options[:rolling] ? KAMAL.traefik_hosts : [ KAMAL.traefik_hosts ]
-        host_list = Array(hosts).join(",")
+        host_groups.each do |hosts|
-        run_hook "pre-traefik-reboot", hosts: host_list
+          host_list = Array(hosts).join(",")
-        on(hosts) do
+          run_hook "pre-traefik-reboot", hosts: host_list
-          execute *KAMAL.auditor.record("Rebooted traefik"), verbosity: :debug
+          on(hosts) do
-          execute *KAMAL.registry.login
+            execute *KAMAL.auditor.record("Rebooted traefik"), verbosity: :debug
-          execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
+            execute *KAMAL.registry.login
-          execute *KAMAL.traefik.remove_container
+            execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
-          execute *KAMAL.traefik.run
+            execute *KAMAL.traefik.remove_container
+            execute *KAMAL.traefik.run
+          end
+          run_hook "post-traefik-reboot", hosts: host_list
         end
-        run_hook "post-traefik-reboot", hosts: host_list
       end
     end
   end
 
   desc "start", "Start existing Traefik container on servers"
   def start
-    mutating do
+    with_lock do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.auditor.record("Started traefik"), verbosity: :debug
         execute *KAMAL.traefik.start
@@ -41,7 +44,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
 
   desc "stop", "Stop existing Traefik container on servers"
   def stop
-    mutating do
+    with_lock do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.auditor.record("Stopped traefik"), verbosity: :debug
         execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
@@ -51,7 +54,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
 
   desc "restart", "Restart existing Traefik container on servers"
   def restart
-    mutating do
+    with_lock do
       stop
       start
     end
@@ -88,7 +91,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
 
   desc "remove", "Remove Traefik container and image from servers"
   def remove
-    mutating do
+    with_lock do
       stop
       remove_container
       remove_image
@@ -97,7 +100,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
 
   desc "remove_container", "Remove Traefik container from servers", hide: true
   def remove_container
-    mutating do
+    with_lock do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.auditor.record("Removed traefik container"), verbosity: :debug
         execute *KAMAL.traefik.remove_container
@@ -107,7 +110,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
 
   desc "remove_image", "Remove Traefik image from servers", hide: true
   def remove_image
-    mutating do
+    with_lock do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.auditor.record("Removed traefik image"), verbosity: :debug
         execute *KAMAL.traefik.remove_image

lib/kamal/commander.rb

@@ -2,12 +2,14 @@ require "active_support/core_ext/enumerable"
 require "active_support/core_ext/module/delegation"
 
 class Kamal::Commander
-  attr_accessor :verbosity, :holding_lock, :hold_lock_on_error
+  attr_accessor :verbosity, :holding_lock, :connected
+  delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :traefik_hosts, :accessory_hosts, to: :specifics
 
   def initialize
     self.verbosity = :info
     self.holding_lock = false
-    self.hold_lock_on_error = false
+    self.connected = false
+    @specifics = nil
   end
 
   def config
@@ -24,10 +26,12 @@ class Kamal::Commander
   attr_reader :specific_roles, :specific_hosts
 
   def specific_primary!
+    @specifics = nil
     self.specific_hosts = [ config.primary_host ]
   end
 
   def specific_roles=(role_names)
+    @specifics = nil
     if role_names.present?
       @specific_roles = Kamal::Utils.filter_specific_items(role_names, config.roles)
 
@@ -40,6 +44,7 @@ class Kamal::Commander
   end
 
   def specific_hosts=(hosts)
+    @specifics = nil
     if hosts.present?
       @specific_hosts = Kamal::Utils.filter_specific_items(hosts, config.all_hosts)
 
@@ -51,39 +56,6 @@ class Kamal::Commander
     end
   end
 
-  def primary_host
-    # Given a list of specific roles, make an effort to match up with the primary_role
-    specific_hosts&.first || specific_roles&.detect { |role| role == config.primary_role }&.primary_host || specific_roles&.first&.primary_host || config.primary_host
-  end
-
-  def primary_role
-    roles_on(primary_host).first
-  end
-
-  def roles
-    (specific_roles || config.roles).select do |role|
-      ((specific_hosts || config.all_hosts) & role.hosts).any?
-    end
-  end
-
-  def hosts
-    (specific_hosts || config.all_hosts).select do |host|
-      (specific_roles || config.roles).flat_map(&:hosts).include?(host)
-    end
-  end
-
-  def roles_on(host)
-    roles.select { |role| role.hosts.include?(host.to_s) }
-  end
-
-  def traefik_hosts
-    specific_hosts || config.traefik_hosts
-  end
-
-  def accessory_hosts
-    specific_hosts || config.accessories.flat_map(&:hosts)
-  end
-
   def accessory_names
     config.accessories&.collect(&:name) || []
   end
@@ -93,8 +65,8 @@ class Kamal::Commander
   end
 
 
-  def app(role: nil)
+  def app(role: nil, host: nil)
-    Kamal::Commands::App.new(config, role: role)
+    Kamal::Commands::App.new(config, role: role, host: host)
   end
 
   def accessory(name)
@@ -166,8 +138,8 @@ class Kamal::Commander
     self.holding_lock
   end
 
-  def hold_lock_on_error?
+  def connected?
-    self.hold_lock_on_error
+    self.connected
   end
 
   private
@@ -181,4 +153,8 @@ class Kamal::Commander
       SSHKit.config.command_map[:docker] = "docker" # No need to use /usr/bin/env, just clogs up the logs
       SSHKit.config.output_verbosity = verbosity
     end
+
+    def specifics
+      @specifics ||= Kamal::Commander::Specifics.new(config, specific_hosts, specific_roles)
+    end
 end

lib/kamal/commander/specifics.rb

@@ -0,0 +1,49 @@
+class Kamal::Commander::Specifics
+  attr_reader :primary_host, :primary_role, :hosts, :roles
+  delegate :stable_sort!, to: Kamal::Utils
+
+  def initialize(config, specific_hosts, specific_roles)
+    @config, @specific_hosts, @specific_roles = config, specific_hosts, specific_roles
+
+    @roles, @hosts = specified_roles, specified_hosts
+
+    @primary_host = specific_hosts&.first || primary_specific_role&.primary_host || config.primary_host
+    @primary_role = primary_or_first_role(roles_on(primary_host))
+
+    stable_sort!(roles) { |role| role == primary_role ? 0 : 1 }
+    stable_sort!(hosts) { |host| roles_on(host).any? { |role| role == primary_role } ? 0 : 1 }
+  end
+
+  def roles_on(host)
+    roles.select { |role| role.hosts.include?(host.to_s) }
+  end
+
+  def traefik_hosts
+    config.traefik_hosts & specified_hosts
+  end
+
+  def accessory_hosts
+    specific_hosts || config.accessories.flat_map(&:hosts)
+  end
+
+  private
+    attr_reader :config, :specific_hosts, :specific_roles
+
+    def primary_specific_role
+      primary_or_first_role(specific_roles) if specific_roles.present?
+    end
+
+    def primary_or_first_role(roles)
+      roles.detect { |role| role == config.primary_role } || roles.first
+    end
+
+    def specified_roles
+      (specific_roles || config.roles) \
+        .select { |role| ((specific_hosts || config.all_hosts) & role.hosts).any? }
+    end
+
+    def specified_hosts
+      (specific_hosts || config.all_hosts) \
+        .select { |host| (specific_roles || config.roles).flat_map(&:hosts).include?(host) }
+    end
+end

lib/kamal/commands/accessory.rb

@@ -99,11 +99,11 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
   end
 
   def make_env_directory
-    make_directory accessory_config.host_env_directory
+    make_directory accessory_config.env.secrets_directory
   end
 
   def remove_env_file
-    [:rm, "-f", accessory_config.host_env_file_path]
+    [ :rm, "-f", accessory_config.env.secrets_file ]
   end
 
   private

lib/kamal/commands/app.rb

@@ -3,11 +3,12 @@ class Kamal::Commands::App < Kamal::Commands::Base
 
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 
-  attr_reader :role, :role
+  attr_reader :role, :host
 
-  def initialize(config, role: nil)
+  def initialize(config, role: nil, host: nil)
     super(config)
     @role = role
+    @host = host
   end
 
   def run(hostname: nil)
@@ -15,10 +16,10 @@ class Kamal::Commands::App < Kamal::Commands::Base
       "--detach",
       "--restart unless-stopped",
       "--name", container_name,
-      *(["--hostname", hostname] if hostname),
+      *([ "--hostname", hostname ] if hostname),
       "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
       "-e", "KAMAL_VERSION=\"#{config.version}\"",
-      *role.env_args,
+      *role.env_args(host),
       *role.health_check_args,
       *role.logging_args,
       *config.volume_args,
@@ -49,7 +50,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
 
 
   def current_running_container_id
-    docker :ps, "--quiet", *filter_args(statuses: ACTIVE_DOCKER_STATUSES), "--latest"
+    current_running_container(format: "--quiet")
   end
 
   def container_id_for_version(version, only_running: false)
@@ -57,22 +58,24 @@ class Kamal::Commands::App < Kamal::Commands::Base
   end
 
   def current_running_version
-    list_versions("--latest", statuses: ACTIVE_DOCKER_STATUSES)
+    pipe \
+      current_running_container(format: "--format '{{.Names}}'"),
+      extract_version_from_name
   end
 
   def list_versions(*docker_args, statuses: nil)
     pipe \
       docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
-      %(while read line; do echo ${line##{role.container_prefix}-}; done) # Extract SHA from "service-role-dest-SHA"
+      extract_version_from_name
   end
 
 
   def make_env_directory
-    make_directory role.host_env_directory
+    make_directory role.env(host).secrets_directory
   end
 
   def remove_env_file
-    [ :rm, "-f", role.host_env_file_path ]
+    [ :rm, "-f", role.env(host).secrets_file ]
   end
 
 
@@ -81,12 +84,31 @@ class Kamal::Commands::App < Kamal::Commands::Base
       [ role.container_prefix, version || config.version ].compact.join("-")
     end
 
+    def latest_image_id
+      docker :image, :ls, *argumentize("--filter", "reference=#{config.latest_image}"), "--format", "'{{.ID}}'"
+    end
+
+    def current_running_container(format:)
+      pipe \
+        shell(chain(latest_image_container(format: format), latest_container(format: format))),
+        [ :head, "-1" ]
+    end
+
+    def latest_image_container(format:)
+      latest_container format: format, filters: [ "ancestor=$(#{latest_image_id.join(" ")})" ]
+    end
+
+    def latest_container(format:, filters: nil)
+      docker :ps, "--latest", *format, *filter_args(statuses: ACTIVE_DOCKER_STATUSES), argumentize("--filter", filters)
+    end
+
     def filter_args(statuses: nil)
       argumentize "--filter", filters(statuses: statuses)
     end
 
-    def service_role_dest
+    def extract_version_from_name
-      [ config.service, role, config.destination ].compact.join("-")
+      # Extract SHA from "service-role-dest-SHA"
+      %(while read line; do echo ${line##{role.container_prefix}-}; done)
     end
 
     def filters(statuses: nil)

lib/kamal/commands/app/assets.rb

@@ -4,8 +4,8 @@ module Kamal::Commands::App::Assets
 
     combine \
       make_directory(role.asset_extracted_path),
-      [*docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true"],
+      [ *docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true" ],
-      docker(:run, "--name", asset_container, "--detach", "--rm", config.latest_image, "sleep 1000000"),
+      docker(:run, "--name", asset_container, "--detach", "--rm", config.absolute_image, "sleep 1000000"),
       docker(:cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_path),
       docker(:stop, "-t 1", asset_container),
       by: "&&"
@@ -17,7 +17,7 @@ module Kamal::Commands::App::Assets
       old_extracted_path, old_volume_path = role.asset_extracted_path(old_version), role.asset_volume(old_version).host_path
     end
 
-    commands = [make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path)]
+    commands = [ make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path) ]
 
     if old_version.present?
       commands << copy_contents(new_extracted_path, old_volume_path, continue_on_error: true)
@@ -46,6 +46,6 @@ module Kamal::Commands::App::Assets
     end
 
     def copy_contents(source, destination, continue_on_error: false)
-      [ :cp, "-rnT", "#{source}", destination, *("|| true" if continue_on_error)]
+      [ :cp, "-rnT", "#{source}", destination, *("|| true" if continue_on_error) ]
     end
 end

lib/kamal/commands/app/containers.rb

@@ -1,4 +1,6 @@
 module Kamal::Commands::App::Containers
+  DOCKER_HEALTH_LOG_FORMAT    = "'{{json .State.Health}}'"
+
   def list_containers
     docker :container, :ls, "--all", *filter_args
   end
@@ -20,4 +22,10 @@ module Kamal::Commands::App::Containers
   def remove_containers
     docker :container, :prune, "--force", *filter_args
   end
+
+  def container_health_log(version:)
+    pipe \
+      container_id_for(container_name: container_name(version)),
+      xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT))
+  end
 end

lib/kamal/commands/app/cord.rb

@@ -2,7 +2,7 @@ module Kamal::Commands::App::Cord
   def cord(version:)
     pipe \
       docker(:inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", container_name(version)),
-      [:awk, "'$2 == \"#{role.cord_volume.container_path}\" {print $1}'"]
+      [ :awk, "'$2 == \"#{role.cord_volume.container_path}\" {print $1}'" ]
   end
 
   def tie_cord(cord)
@@ -17,6 +17,6 @@ module Kamal::Commands::App::Cord
     def create_empty_file(file)
       chain \
         make_directory_for(file),
-        [:touch, file]
+        [ :touch, file ]
     end
 end

lib/kamal/commands/app/execution.rb

@@ -1,27 +1,29 @@
 module Kamal::Commands::App::Execution
-  def execute_in_existing_container(*command, interactive: false)
+  def execute_in_existing_container(*command, interactive: false, env:)
     docker :exec,
       ("-it" if interactive),
+      *argumentize("--env", env),
       container_name,
       *command
   end
 
-  def execute_in_new_container(*command, interactive: false)
+  def execute_in_new_container(*command, interactive: false, env:)
     docker :run,
       ("-it" if interactive),
       "--rm",
-      *role&.env_args,
+      *role&.env_args(host),
+      *argumentize("--env", env),
       *config.volume_args,
       *role&.option_args,
       config.absolute_image,
       *command
   end
 
-  def execute_in_existing_container_over_ssh(*command, host:)
+  def execute_in_existing_container_over_ssh(*command,  env:)
-    run_over_ssh execute_in_existing_container(*command, interactive: true), host: host
+    run_over_ssh execute_in_existing_container(*command, interactive: true, env: env), host: host
   end
 
-  def execute_in_new_container_over_ssh(*command, host:)
+  def execute_in_new_container_over_ssh(*command, env:)
-    run_over_ssh execute_in_new_container(*command, interactive: true), host: host
+    run_over_ssh execute_in_new_container(*command, interactive: true, env: env), host: host
   end
 end

lib/kamal/commands/app/images.rb

@@ -7,7 +7,7 @@ module Kamal::Commands::App::Images
     docker :image, :prune, "--all", "--force", *filter_args
   end
 
-  def tag_current_image_as_latest
+  def tag_latest_image
     docker :tag, config.absolute_image, config.latest_image
   end
 end

lib/kamal/commands/app/logging.rb

@@ -1,7 +1,7 @@
 module Kamal::Commands::App::Logging
-  def logs(since: nil, lines: nil, grep: nil)
+  def logs(version: nil, since: nil, lines: nil, grep: nil)
     pipe \
-      current_running_container_id,
+      version ? container_id_for_version(version) : current_running_container_id,
       "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
       ("grep '#{grep}'" if grep)
   end

lib/kamal/commands/auditor.rb

@@ -21,7 +21,7 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
     def audit_log_file
       file = [ config.service, config.destination, "audit.log" ].compact.join("-")
 
-      "#{config.run_directory}/#{file}"
+      File.join(config.run_directory, file)
     end
 
     def audit_tags(**details)

lib/kamal/commands/base.rb

@@ -3,7 +3,6 @@ module Kamal::Commands
     delegate :sensitive, :argumentize, to: Kamal::Utils
 
     DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'"
-    DOCKER_HEALTH_LOG_FORMAT    = "'{{json .State.Health}}'"
 
     attr_accessor :config
 
@@ -18,7 +17,7 @@ module Kamal::Commands
         elsif config.ssh.proxy && config.ssh.proxy.is_a?(Net::SSH::Proxy::Command)
           cmd << " -o ProxyCommand='#{config.ssh.proxy.command_line_template}'"
         end
-        cmd << " -t #{config.ssh.user}@#{host} -p #{config.ssh.port} '#{command.join(" ")}'"
+        cmd << " -t #{config.ssh.user}@#{host} -p #{config.ssh.port} '#{command.join(" ").gsub("'", "'\\\\''")}'"
       end
     end
 
@@ -71,13 +70,17 @@ module Kamal::Commands
       end
 
       def shell(command)
-        [ :sh, "-c", "'#{command.flatten.join(" ").gsub("'", "'\\''")}'" ]
+        [ :sh, "-c", "'#{command.flatten.join(" ").gsub("'", "'\\\\''")}'" ]
       end
 
       def docker(*args)
         args.compact.unshift :docker
       end
 
+      def git(*args, path: nil)
+        [ :git, *([ "-C", path ] if path), *args.compact ]
+      end
+
       def tags(**details)
         Kamal::Tags.from_config(config, **details)
       end

lib/kamal/commands/builder.rb

@@ -3,6 +3,8 @@ require "active_support/core_ext/string/filters"
 class Kamal::Commands::Builder < Kamal::Commands::Base
   delegate :create, :remove, :push, :clean, :pull, :info, :validate_image, to: :target
 
+  include Clone
+
   def name
     target.class.to_s.remove("Kamal::Commands::Builder::").underscore.inquiry
   end

lib/kamal/commands/builder/base.rb

@@ -3,7 +3,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
   class BuilderError < StandardError; end
 
   delegate :argumentize, to: Kamal::Utils
-  delegate :args, :secrets, :dockerfile, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, :ssh, to: :builder_config
+  delegate :args, :secrets, :dockerfile, :target, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, :ssh, to: :builder_config
 
   def clean
     docker :image, :rm, "--force", config.absolute_image
@@ -14,7 +14,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
   end
 
   def build_options
-    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_ssh ]
+    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_target, *build_ssh ]
   end
 
   def build_context
@@ -25,7 +25,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
     pipe \
       docker(:inspect, "-f", "'{{ .Config.Labels.service }}'", config.absolute_image),
       any(
-        [:grep, "-x", config.service],
+        [ :grep, "-x", config.service ],
         "(echo \"Image #{config.absolute_image} is missing the 'service' label\" && exit 1)"
       )
   end
@@ -38,8 +38,8 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
 
     def build_cache
       if cache_to && cache_from
-        ["--cache-to", cache_to,
+        [ "--cache-to", cache_to,
-          "--cache-from", cache_from]
+          "--cache-from", cache_from ]
       end
     end
 
@@ -63,6 +63,10 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
       end
     end
 
+    def build_target
+      argumentize "--target", target if target.present?
+    end
+
     def build_ssh
       argumentize "--ssh", ssh if ssh.present?
     end

lib/kamal/commands/builder/clone.rb

@@ -0,0 +1,28 @@
+module Kamal::Commands::Builder::Clone
+  extend ActiveSupport::Concern
+
+  included do
+    delegate :clone_directory, :build_directory, to: :"config.builder"
+  end
+
+  def clone
+    git :clone, Kamal::Git.root, path: clone_directory
+  end
+
+  def clone_reset_steps
+    [
+      git(:remote, "set-url", :origin, Kamal::Git.root, path: build_directory),
+      git(:fetch, :origin, path: build_directory),
+      git(:reset, "--hard", Kamal::Git.revision, path: build_directory),
+      git(:clean, "-fdx", path: build_directory)
+    ]
+  end
+
+  def clone_status
+    git :status, "--porcelain", path: build_directory
+  end
+
+  def clone_revision
+    git :"rev-parse", :HEAD, path: build_directory
+  end
+end

lib/kamal/commands/builder/multiarch.rb

@@ -7,6 +7,12 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
     docker :buildx, :rm, builder_name
   end
 
+  def info
+    combine \
+      docker(:context, :ls),
+      docker(:buildx, :ls)
+  end
+
   def push
     docker :buildx, :build,
       "--push",
@@ -16,12 +22,6 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
       build_context
   end
 
-  def info
-    combine \
-      docker(:context, :ls),
-      docker(:buildx, :ls)
-  end
-
   private
     def builder_name
       "kamal-#{config.service}-multiarch"

lib/kamal/commands/builder/native.rb

@@ -7,14 +7,14 @@ class Kamal::Commands::Builder::Native < Kamal::Commands::Builder::Base
     # No-op on native without cache
   end
 
+  def info
+    # No-op on native
+  end
+
   def push
     combine \
       docker(:build, *build_options, build_context),
       docker(:push, config.absolute_image),
       docker(:push, config.latest_image)
   end
-
-  def info
-    # No-op on native
-  end
 end

lib/kamal/commands/builder/native/remote.rb

@@ -11,21 +11,21 @@ class Kamal::Commands::Builder::Native::Remote < Kamal::Commands::Builder::Nativ
       remove_buildx
   end
 
-  def push
-    docker :buildx, :build,
-      "--push",
-      "--platform", platform,
-      "--builder", builder_name,
-      *build_options,
-      build_context
-  end
-
   def info
     chain \
       docker(:context, :ls),
       docker(:buildx, :ls)
   end
 
+  def push
+    docker :buildx, :build,
+    "--push",
+    "--platform", platform,
+    "--builder", builder_name,
+    *build_options,
+    build_context
+  end
+
 
   private
     def builder_name

lib/kamal/commands/healthcheck.rb

@@ -1,60 +0,0 @@
-class Kamal::Commands::Healthcheck < Kamal::Commands::Base
-
-  def run
-    primary = config.role(config.primary_role)
-
-    docker :run,
-      "--detach",
-      "--name", container_name_with_version,
-      "--publish", "#{exposed_port}:#{config.healthcheck["port"]}",
-      "--label", "service=#{config.healthcheck_service}",
-      "-e", "KAMAL_CONTAINER_NAME=\"#{config.healthcheck_service}\"",
-      *primary.env_args,
-      *primary.health_check_args(cord: false),
-      *config.volume_args,
-      *primary.option_args,
-      config.absolute_image,
-      primary.cmd
-  end
-
-  def status
-    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_STATUS_FORMAT))
-  end
-
-  def container_health_log
-    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT))
-  end
-
-  def logs
-    pipe container_id, xargs(docker(:logs, "--tail", log_lines, "2>&1"))
-  end
-
-  def stop
-    pipe container_id, xargs(docker(:stop))
-  end
-
-  def remove
-    pipe container_id, xargs(docker(:container, :rm))
-  end
-
-  private
-    def container_name_with_version
-      "#{config.healthcheck_service}-#{config.version}"
-    end
-
-    def container_id
-      container_id_for(container_name: container_name_with_version)
-    end
-
-    def health_url
-      "http://localhost:#{exposed_port}#{config.healthcheck["path"]}"
-    end
-
-    def exposed_port
-      config.healthcheck["exposed_port"]
-    end
-
-    def log_lines
-      config.healthcheck["log_lines"]
-    end
-end

lib/kamal/commands/hook.rb

@@ -9,6 +9,6 @@ class Kamal::Commands::Hook < Kamal::Commands::Base
 
   private
     def hook_file(hook)
-      "#{config.hooks_path}/#{hook}"
+      File.join(config.hooks_path, hook)
     end
 end

lib/kamal/commands/lock.rb

@@ -5,14 +5,14 @@ require "base64"
 class Kamal::Commands::Lock < Kamal::Commands::Base
   def acquire(message, version)
     combine \
-      [:mkdir, lock_dir],
+      [ :mkdir, lock_dir ],
       write_lock_details(message, version)
   end
 
   def release
     combine \
-      [:rm, lock_details_file],
+      [ :rm, lock_details_file ],
-      [:rm, "-r", lock_dir]
+      [ :rm, "-r", lock_dir ]
   end
 
   def status
@@ -21,31 +21,41 @@ class Kamal::Commands::Lock < Kamal::Commands::Base
       read_lock_details
   end
 
+  def ensure_locks_directory
+    [ :mkdir, "-p", locks_dir ]
+  end
+
   private
     def write_lock_details(message, version)
       write \
-        [:echo, "\"#{Base64.encode64(lock_details(message, version))}\""],
+        [ :echo, "\"#{Base64.encode64(lock_details(message, version))}\"" ],
         lock_details_file
     end
 
     def read_lock_details
       pipe \
-        [:cat, lock_details_file],
+        [ :cat, lock_details_file ],
-        [:base64, "-d"]
+        [ :base64, "-d" ]
     end
 
     def stat_lock_dir
       write \
-        [:stat, lock_dir],
+        [ :stat, lock_dir ],
         "/dev/null"
     end
 
+    def locks_dir
+      File.join(config.run_directory, "locks")
+    end
+
     def lock_dir
-      "#{config.run_directory}/lock-#{config.service}"
+      dir_name = [ config.service, config.destination ].compact.join("-")
+
+      File.join(locks_dir, dir_name)
     end
 
     def lock_details_file
-      [lock_dir, :details].join("/")
+      File.join(lock_dir, "details")
     end
 
     def lock_details(message, version)

lib/kamal/commands/prune.rb

@@ -26,7 +26,7 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
 
   private
     def stopped_containers_filters
-      [ "created", "exited", "dead" ].flat_map { |status| ["--filter", "status=#{status}"] }
+      [ "created", "exited", "dead" ].flat_map { |status| [ "--filter", "status=#{status}" ] }
     end
 
     def active_image_list
@@ -43,4 +43,4 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
     def healthcheck_service_filter
       [ "--filter", "label=service=#{config.healthcheck_service}" ]
     end
-  end
+end

lib/kamal/commands/server.rb

@@ -1,5 +1,5 @@
 class Kamal::Commands::Server < Kamal::Commands::Base
   def ensure_run_directory
-    [:mkdir, "-p", config.run_directory]
+    [ :mkdir, "-p", config.run_directory ]
   end
 end

lib/kamal/commands/traefik.rb

@@ -4,7 +4,7 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
   DEFAULT_IMAGE = "traefik:v2.10"
   CONTAINER_PORT = 80
   DEFAULT_ARGS = {
-    'log.level' => 'DEBUG'
+    "log.level" => "DEBUG"
   }
   DEFAULT_LABELS = {
     # These ensure we serve a 502 rather than a 404 if no containers are available
@@ -71,20 +71,18 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
     "#{host_port}:#{CONTAINER_PORT}"
   end
 
-  def env_file
+  def env
-    Kamal::EnvFile.new(config.traefik.fetch("env", {}))
+    Kamal::Configuration::Env.from_config \
-  end
+      config: config.traefik.fetch("env", {}),
-
+      secrets_file: File.join(config.host_env_directory, "traefik", "traefik.env")
-  def host_env_file_path
-    File.join host_env_directory, "traefik.env"
   end
 
   def make_env_directory
-    make_directory(host_env_directory)
+    make_directory(env.secrets_directory)
   end
 
   def remove_env_file
-    [:rm, "-f", host_env_file_path]
+    [ :rm, "-f", env.secrets_file ]
   end
 
   private
@@ -97,11 +95,7 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
     end
 
     def env_args
-      argumentize "--env-file", host_env_file_path
+      env.args
-    end
-
-    def host_env_directory
-      File.join config.host_env_directory, "traefik"
     end
 
     def labels

lib/kamal/configuration.rb

@@ -6,7 +6,7 @@ require "erb"
 require "net/ssh/proxy/jump"
 
 class Kamal::Configuration
-  delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, :logging, to: :raw_config, allow_nil: true
+  delegate :service, :image, :servers, :labels, :registry, :stop_wait_time, :hooks_path, :logging, to: :raw_config, allow_nil: true
   delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_reader :destination, :raw_config
@@ -88,7 +88,7 @@ class Kamal::Configuration
 
 
   def all_hosts
-    roles.flat_map(&:hosts).uniq
+    (roles + accessories).flat_map(&:hosts).uniq
   end
 
   def primary_host
@@ -128,7 +128,11 @@ class Kamal::Configuration
   end
 
   def latest_image
-    "#{repository}:latest"
+    "#{repository}:#{latest_tag}"
+  end
+
+  def latest_tag
+    [ "latest", *destination ].join("-")
   end
 
   def service_with_version
@@ -184,7 +188,7 @@ class Kamal::Configuration
 
 
   def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }.merge(raw_config.healthcheck || {})
+    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }.merge(raw_config.healthcheck || {})
   end
 
   def healthcheck_service
@@ -216,15 +220,32 @@ class Kamal::Configuration
     raw_config.hooks_path || ".kamal/hooks"
   end
 
-  def host_env_directory
-    "#{run_directory}/env"
-  end
-
   def asset_path
     raw_config.asset_path
   end
 
 
+  def host_env_directory
+    File.join(run_directory, "env")
+  end
+
+  def env
+    raw_config.env || {}
+  end
+
+  def env_tags
+    @env_tags ||= if (tags = raw_config.env["tags"])
+      tags.collect { |name, config| Kamal::Configuration::Env::Tag.new(name, config: config) }
+    else
+      []
+    end
+  end
+
+  def env_tag(name)
+    env_tags.detect { |t| t.name == name.to_s }
+  end
+
+
   def valid?
     ensure_destination_if_required && ensure_required_keys_present && ensure_valid_kamal_version && ensure_retain_containers_valid && ensure_valid_service_name
   end
@@ -292,7 +313,7 @@ class Kamal::Configuration
     end
 
     def ensure_valid_service_name
-      raise ArgumentError, "Service name can only include alphanumeric characters, hyphens, and underscores" unless raw_config[:service] =~ /^[a-z0-9_-]+$/
+      raise ArgumentError, "Service name can only include alphanumeric characters, hyphens, and underscores" unless raw_config[:service] =~ /^[a-z0-9_-]+$/i
 
       true
     end
@@ -319,7 +340,10 @@ class Kamal::Configuration
     def git_version
       @git_version ||=
         if Kamal::Git.used?
-          [ Kamal::Git.revision, Kamal::Git.uncommitted_changes.present? ? "_uncommitted_#{SecureRandom.hex(8)}" : "" ].join
+          if Kamal::Git.uncommitted_changes.present? && !builder.git_clone?
+            uncommitted_suffix = "_uncommitted_#{SecureRandom.hex(8)}"
+          end
+          [ Kamal::Git.revision, uncommitted_suffix ].compact.join
         else
           raise "Can't use commit hash as version, no git repository found in #{Dir.pwd}"
         end

lib/kamal/configuration/accessory.rb

@@ -16,7 +16,7 @@ class Kamal::Configuration::Accessory
   end
 
   def hosts
-    if (specifics.keys & ["host", "hosts", "roles"]).size != 1
+    if (specifics.keys & [ "host", "hosts", "roles" ]).size != 1
       raise ArgumentError, "Specify one of `host`, `hosts` or `roles` for accessory `#{name}`"
     end
 
@@ -42,23 +42,13 @@ class Kamal::Configuration::Accessory
   end
 
   def env
-    specifics["env"] || {}
+    Kamal::Configuration::Env.from_config \
-  end
+      config: specifics.fetch("env", {}),
-
+      secrets_file: File.join(config.host_env_directory, "accessories", "#{service_name}.env")
-  def env_file
-    Kamal::EnvFile.new(env)
-  end
-
-  def host_env_directory
-    File.join config.host_env_directory, "accessories"
-  end
-
-  def host_env_file_path
-    File.join host_env_directory, "#{service_name}.env"
   end
 
   def env_args
-    argumentize "--env-file", host_env_file_path
+    env.args
   end
 
   def files
@@ -111,10 +101,10 @@ class Kamal::Configuration::Accessory
     end
 
     def with_clear_env_loaded
-      (env["clear"] || env).each { |k, v| ENV[k] = v }
+      env.clear.each { |k, v| ENV[k] = v }
       yield
     ensure
-      (env["clear"] || env).each { |k, v| ENV.delete(k) }
+      env.clear.each { |k, v| ENV.delete(k) }
     end
 
     def read_dynamic_file(local_file)
@@ -144,7 +134,7 @@ class Kamal::Configuration::Accessory
     end
 
     def expand_host_path(host_path)
-      absolute_path?(host_path) ? host_path : "#{service_data_directory}/#{host_path}"
+      absolute_path?(host_path) ? host_path : File.join(service_data_directory, host_path)
     end
 
     def absolute_path?(path)
@@ -159,7 +149,7 @@ class Kamal::Configuration::Accessory
       if specifics.key?("host")
         host = specifics["host"]
         if host
-          [host]
+          [ host ]
         else
           raise ArgumentError, "Missing host for accessory `#{name}`"
         end

lib/kamal/configuration/boot.rb

@@ -8,7 +8,7 @@ class Kamal::Configuration::Boot
     limit = @options["limit"]
 
     if limit.to_s.end_with?("%")
-      [@host_count * limit.to_i / 100, 1].max
+      [ @host_count * limit.to_i / 100, 1 ].max
     else
       limit
     end

lib/kamal/configuration/builder.rb

@@ -3,6 +3,8 @@ class Kamal::Configuration::Builder
     @options = config.raw_config.builder || {}
     @image = config.image
     @server = config.registry["server"]
+    @service = config.service
+    @destination = config.destination
 
     valid?
   end
@@ -39,6 +41,10 @@ class Kamal::Configuration::Builder
     @options["dockerfile"] || "Dockerfile"
   end
 
+  def target
+    @options["target"]
+  end
+
   def context
     @options["context"] || "."
   end
@@ -85,10 +91,27 @@ class Kamal::Configuration::Builder
     @options["ssh"]
   end
 
+  def git_clone?
+    Kamal::Git.used? && @options["context"].nil?
+  end
+
+  def clone_directory
+    @clone_directory ||= File.join Dir.tmpdir, "kamal-clones", [ @service, pwd_sha ].compact.join("-")
+  end
+
+  def build_directory
+    @build_directory ||=
+      if git_clone?
+        File.join clone_directory, repo_basename, repo_relative_pwd
+      else
+        "."
+      end
+  end
+
   private
     def valid?
       if @options["cache"] && @options["cache"]["type"]
-        raise ArgumentError, "Invalid cache type: #{@options["cache"]["type"]}" unless ["gha", "registry"].include?(@options["cache"]["type"])
+        raise ArgumentError, "Invalid cache type: #{@options["cache"]["type"]}" unless [ "gha", "registry" ].include?(@options["cache"]["type"])
       end
     end
 
@@ -109,10 +132,22 @@ class Kamal::Configuration::Builder
     end
 
     def cache_to_config_for_gha
-      [ "type=gha", @options["cache"]&.fetch("options", nil)].compact.join(",")
+      [ "type=gha", @options["cache"]&.fetch("options", nil) ].compact.join(",")
     end
 
     def cache_to_config_for_registry
       [ "type=registry", @options["cache"]&.fetch("options", nil), "ref=#{cache_image_ref}" ].compact.join(",")
     end
+
+    def repo_basename
+      File.basename(Kamal::Git.root)
+    end
+
+    def repo_relative_pwd
+      Dir.pwd.delete_prefix(Kamal::Git.root)
+    end
+
+    def pwd_sha
+      Digest::SHA256.hexdigest(Dir.pwd)[0..12]
+    end
 end

lib/kamal/configuration/env.rb

@@ -0,0 +1,40 @@
+class Kamal::Configuration::Env
+  attr_reader :secrets_keys, :clear, :secrets_file
+  delegate :argumentize, to: Kamal::Utils
+
+  def self.from_config(config:, secrets_file: nil)
+    secrets_keys = config.fetch("secret", [])
+    clear = config.fetch("clear", config.key?("secret") || config.key?("tags") ? {} : config)
+
+    new clear: clear, secrets_keys: secrets_keys, secrets_file: secrets_file
+  end
+
+  def initialize(clear:, secrets_keys:, secrets_file:)
+    @clear = clear
+    @secrets_keys = secrets_keys
+    @secrets_file = secrets_file
+  end
+
+  def args
+    [ "--env-file", secrets_file, *argumentize("--env", clear) ]
+  end
+
+  def secrets_io
+    StringIO.new(Kamal::EnvFile.new(secrets).to_s)
+  end
+
+  def secrets
+    @secrets ||= secrets_keys.to_h { |key| [ key, ENV.fetch(key) ] }
+  end
+
+  def secrets_directory
+    File.dirname(secrets_file)
+  end
+
+  def merge(other)
+    self.class.new \
+      clear: @clear.merge(other.clear),
+      secrets_keys: @secrets_keys | other.secrets_keys,
+      secrets_file: secrets_file
+  end
+end

lib/kamal/configuration/env/tag.rb

@@ -0,0 +1,12 @@
+class Kamal::Configuration::Env::Tag
+  attr_reader :name, :config
+
+  def initialize(name, config:)
+    @name = name
+    @config = config
+  end
+
+  def env
+    Kamal::Configuration::Env.from_config(config: config)
+  end
+end

lib/kamal/configuration/role.rb

@@ -7,6 +7,7 @@ class Kamal::Configuration::Role
 
   def initialize(name, config:)
     @name, @config = name.inquiry, config
+    @tagged_hosts ||= extract_tagged_hosts_from_config
   end
 
   def primary_host
@@ -14,7 +15,11 @@ class Kamal::Configuration::Role
   end
 
   def hosts
-    @hosts ||= extract_hosts_from_config
+    tagged_hosts.keys
+  end
+
+  def env_tags(host)
+    tagged_hosts.fetch(host).collect { |tag| config.env_tag(tag) }
   end
 
   def cmd
@@ -50,28 +55,13 @@ class Kamal::Configuration::Role
   end
 
 
-  def env
+  def env(host)
-    if config.env && config.env["secret"]
+    @envs ||= {}
-      merged_env_with_secrets
+    @envs[host] ||= [ base_env, specialized_env, *env_tags(host).map(&:env) ].reduce(:merge)
-    else
-      merged_env
-    end
   end
 
-  def env_file
+  def env_args(host)
-    Kamal::EnvFile.new(env)
+    env(host).args
-  end
-
-  def host_env_directory
-    File.join config.host_env_directory, "roles"
-  end
-
-  def host_env_file_path
-    File.join host_env_directory, "#{[config.service, name, config.destination].compact.join("-")}.env"
-  end
-
-  def env_args
-    argumentize "--env-file", host_env_file_path
   end
 
   def asset_volume_args
@@ -123,13 +113,13 @@ class Kamal::Configuration::Role
   end
 
   def cord_host_directory
-    File.join config.run_directory_as_docker_volume, "cords", [container_prefix, config.run_id].join("-")
+    File.join config.run_directory_as_docker_volume, "cords", [ container_prefix, config.run_id ].join("-")
   end
 
   def cord_volume
     if (cord = health_check_options["cord"])
       @cord_volume ||= Kamal::Configuration::Volume.new \
-        host_path: File.join(config.run_directory, "cords", [container_prefix, config.run_id].join("-")),
+        host_path: File.join(config.run_directory, "cords", [ container_prefix, config.run_id ].join("-")),
         container_path: cord
     end
   end
@@ -180,7 +170,24 @@ class Kamal::Configuration::Role
   end
 
   private
-    attr_accessor :config
+    attr_accessor :config, :tagged_hosts
+
+    def extract_tagged_hosts_from_config
+      {}.tap do |tagged_hosts|
+        extract_hosts_from_config.map do |host_config|
+          if host_config.is_a?(Hash)
+            raise ArgumentError, "Multiple hosts found: #{host_config.inspect}" unless host_config.size == 1
+
+            host, tags = host_config.first
+            tagged_hosts[host] = Array(tags)
+          elsif host_config.is_a?(String) || host_config.is_a?(Symbol)
+            tagged_hosts[host_config] = []
+          else
+            raise ArgumentError, "Invalid host config: #{host_config.inspect}"
+          end
+        end
+      end
+    end
 
     def extract_hosts_from_config
       if config.servers.is_a?(Array)
@@ -192,11 +199,7 @@ class Kamal::Configuration::Role
     end
 
     def default_labels
-      if config.destination
+      { "service" => config.service, "role" => name, "destination" => config.destination }
-        { "service" => config.service, "role" => name, "destination" => config.destination }
-      else
-        { "service" => config.service, "role" => name }
-      end
     end
 
     def traefik_labels
@@ -217,7 +220,7 @@ class Kamal::Configuration::Role
     end
 
     def traefik_service
-      [ config.service, name, config.destination ].compact.join("-")
+      container_prefix
     end
 
     def custom_labels
@@ -229,31 +232,21 @@ class Kamal::Configuration::Role
 
     def specializations
       if config.servers.is_a?(Array) || config.servers[name].is_a?(Array)
-        { }
+        {}
       else
         config.servers[name].except("hosts")
       end
     end
 
     def specialized_env
-      specializations["env"] || {}
+      Kamal::Configuration::Env.from_config config: specializations.fetch("env", {})
-    end
-
-    def merged_env
-      config.env&.merge(specialized_env) || {}
     end
 
     # Secrets are stored in an array, which won't merge by default, so have to do it by hand.
-    def merged_env_with_secrets
+    def base_env
-      merged_env.tap do |new_env|
+      Kamal::Configuration::Env.from_config \
-        new_env["secret"] = Array(config.env["secret"]) + Array(specialized_env["secret"])
+        config: config.env,
-
+        secrets_file: File.join(config.host_env_directory, "roles", "#{container_prefix}.env")
-        # If there's no secret/clear split, everything is clear
-        clear_app_env  = config.env["secret"] ? Array(config.env["clear"]) : Array(config.env["clear"] || config.env)
-        clear_role_env = specialized_env["secret"] ? Array(specialized_env["clear"]) : Array(specialized_env["clear"] || specialized_env)
-
-        new_env["clear"] = clear_app_env.to_h.merge(clear_role_env.to_h)
-      end
     end
 
     def http_health_check(port:, path:)

lib/kamal/env_file.rb

@@ -6,18 +6,8 @@ class Kamal::EnvFile
 
   def to_s
     env_file = StringIO.new.tap do |contents|
-      if (secrets = @env["secret"]).present?
+      @env.each do |key, value|
-        @env.fetch("secret", @env)&.each do |key|
+        contents << docker_env_file_line(key, value)
-          contents << docker_env_file_line(key, ENV.fetch(key))
-        end
-
-        @env["clear"]&.each do |key, value|
-          contents << docker_env_file_line(key, value)
-        end
-      else
-        @env.fetch("clear", @env)&.each do |key, value|
-          contents << docker_env_file_line(key, value)
-        end
       end
     end.string
 
@@ -29,11 +19,18 @@ class Kamal::EnvFile
 
   private
     def docker_env_file_line(key, value)
-      "#{key.to_s}=#{escape_docker_env_file_value(value)}\n"
+      "#{key}=#{escape_docker_env_file_value(value)}\n"
     end
 
     # Escape a value to make it safe to dump in a docker file.
     def escape_docker_env_file_value(value)
+      # keep non-ascii(UTF-8) characters as it is
+      value.to_s.scan(/[\x00-\x7F]+|[^\x00-\x7F]+/).map do |part|
+        part.ascii_only? ? escape_docker_env_file_ascii_value(part) : part
+      end.join
+    end
+
+    def escape_docker_env_file_ascii_value(value)
       # Doublequotes are treated literally in docker env files
       # so remove leading and trailing ones and unescape any others
       value.to_s.dump[1..-2].gsub(/\\"/, "\"")

lib/kamal/git.rb

@@ -16,4 +16,8 @@ module Kamal::Git
   def uncommitted_changes
     `git status --porcelain`.strip
   end
+
+  def root
+    `git rev-parse --show-toplevel`.strip
+  end
 end

lib/kamal/sshkit_with_ext.rb

@@ -103,3 +103,39 @@ class SSHKit::Backend::Netssh
 
   prepend LimitConcurrentStartsInstance
 end
+
+class SSHKit::Runner::Parallel
+  # SSHKit joins the threads in sequence and fails on the first error it encounters, which means that we wait threads
+  # before the first failure to complete but not for ones after.
+  #
+  # We'll patch it to wait for them all to complete, and to record all the threads that errored so we can see when a
+  # problem occurs on multiple hosts.
+  module CompleteAll
+    def execute
+      threads = hosts.map do |host|
+        Thread.new(host) do |h|
+          backend(h, &block).run
+        rescue ::StandardError => e
+          e2 = SSHKit::Runner::ExecuteError.new e
+          raise e2, "Exception while executing #{host.user ? "as #{host.user}@" : "on host "}#{host}: #{e.message}"
+        end
+      end
+
+      exceptions = []
+      threads.each do |t|
+        begin
+          t.join
+        rescue SSHKit::Runner::ExecuteError => e
+          exceptions << e
+        end
+      end
+      if exceptions.one?
+        raise exceptions.first
+      elsif exceptions.many?
+        raise exceptions.first, [ "Exceptions on #{exceptions.count} hosts:", exceptions.map(&:message) ].join("\n")
+      end
+    end
+  end
+
+  prepend CompleteAll
+end

lib/kamal/utils.rb

@@ -9,7 +9,7 @@ module Kamal::Utils
       if value.present?
         attr = "#{key}=#{escape_shell_value(value)}"
         attr = self.sensitive(attr, redaction: "#{key}=[REDACTED]") if sensitive
-        [ argument, attr]
+        [ argument, attr ]
       else
         [ argument, key ]
       end
@@ -29,7 +29,7 @@ module Kamal::Utils
 
   # Flattens a one-to-many structure into an array of two-element arrays each containing a key-value pair
   def flatten_args(args)
-    args.flat_map { |key, value| value.try(:map) { |entry| [key, entry] } || [ [ key, value ] ] }
+    args.flat_map { |key, value| value.try(:map) { |entry| [ key, entry ] } || [ [ key, value ] ] }
   end
 
   # Marks sensitive values for redaction in logs and human-visible output.
@@ -66,12 +66,15 @@ module Kamal::Utils
     Array(filters).select do |filter|
       matches += Array(items).select do |item|
         # Only allow * for a wildcard
-        pattern = Regexp.escape(filter).gsub('\*', '.*')
         # items are roles or hosts
-        (item.respond_to?(:name) ? item.name : item).match(/^#{pattern}$/)
+        File.fnmatch(filter, item.to_s, File::FNM_EXTGLOB)
       end
     end
 
-    matches
+    matches.uniq
+  end
+
+  def stable_sort!(elements, &block)
+    elements.sort_by!.with_index { |element, index| [ block.call(element), index ] }
   end
 end

lib/kamal/version.rb

@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "1.4.0"
+  VERSION = "1.6.0"
 end

test/cli/accessory_test.rb

@@ -7,7 +7,7 @@ class CliAccessoryTest < CliTestCase
 
     run_command("boot", "mysql").tap do |output|
       assert_match /docker login.*on 1.1.1.3/, output
-      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
     end
   end
 
@@ -21,7 +21,7 @@ class CliAccessoryTest < CliTestCase
       assert_match /docker login.*on 1.1.1.3/, output
       assert_match /docker login.*on 1.1.1.1/, output
       assert_match /docker login.*on 1.1.1.2/, output
-      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
       assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
       assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
     end
@@ -76,11 +76,20 @@ class CliAccessoryTest < CliTestCase
   end
 
   test "details" do
-    assert_match "docker ps --filter label=service=app-mysql", run_command("details", "mysql")
+    run_command("details", "mysql").tap do |output|
+      assert_match "docker ps --filter label=service=app-mysql", output
+      assert_match "Accessory mysql Host: 1.1.1.3", output
+    end
+  end
+
+  test "details with non-existent accessory" do
+    assert_equal "No accessory by the name of 'hello' (options: mysql and redis)", stderred { run_command("details", "hello") }
   end
 
   test "details with all" do
     run_command("details", "all").tap do |output|
+      assert_match "Accessory mysql Host: 1.1.1.3", output
+      assert_match "Accessory redis Host: 1.1.1.2", output
       assert_match "docker ps --filter label=service=app-mysql", output
       assert_match "docker ps --filter label=service=app-redis", output
     end
@@ -154,9 +163,9 @@ class CliAccessoryTest < CliTestCase
 
     run_command("boot", "redis", "--hosts", "1.1.1.1").tap do |output|
       assert_match /docker login.*on 1.1.1.1/, output
-      refute_match /docker login.*on 1.1.1.2/, output
+      assert_no_match /docker login.*on 1.1.1.2/, output
       assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
-      refute_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
+      assert_no_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
     end
   end
 
@@ -166,14 +175,14 @@ class CliAccessoryTest < CliTestCase
 
     run_command("boot", "redis", "--hosts", "1.1.1.1,1.1.1.3").tap do |output|
       assert_match /docker login.*on 1.1.1.1/, output
-      refute_match /docker login.*on 1.1.1.3/, output
+      assert_no_match /docker login.*on 1.1.1.3/, output
       assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
-      refute_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.3", output
+      assert_no_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.3", output
     end
   end
 
   private
     def run_command(*command)
-      stdouted { Kamal::Cli::Accessory.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Accessory.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
     end
 end

test/cli/app_test.rb

@@ -23,11 +23,11 @@ class CliAppTest < CliTestCase
       .returns("running") # health check
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
       .returns("123") # old version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
       .returns("cordfile") # old version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -45,7 +45,7 @@ class CliAppTest < CliTestCase
   end
 
   test "boot uses group strategy when specified" do
-    Kamal::Cli::App.any_instance.stubs(:on).with("1.1.1.1").twice # acquire & release lock
+    Kamal::Cli::App.any_instance.stubs(:on).with("1.1.1.1").times(3) # ensure locks dir, acquire & release lock
     Kamal::Cli::App.any_instance.stubs(:on).with([ "1.1.1.1" ]) # tag container
 
     # Strategy is used when booting the containers
@@ -54,14 +54,14 @@ class CliAppTest < CliTestCase
     run_command("boot", config: :with_boot_strategy)
   end
 
-  test "boot errors leave lock in place" do
+  test "boot errors don't leave lock in place" do
     Kamal::Cli::App.any_instance.expects(:using_version).raises(RuntimeError)
 
-    assert !KAMAL.holding_lock?
+    assert_not KAMAL.holding_lock?
     assert_raises(RuntimeError) do
       stderred { run_command("boot") }
     end
-    assert KAMAL.holding_lock?
+    assert_not KAMAL.holding_lock?
   end
 
   test "boot with assets" do
@@ -75,11 +75,11 @@ class CliAppTest < CliTestCase
       .returns("running") # health check
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
       .returns("123").twice # old version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
       .returns("") # old version
 
     run_command("boot", config: :with_assets).tap do |output|
@@ -92,6 +92,82 @@ class CliAppTest < CliTestCase
     end
   end
 
+  test "boot with host tags" do
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+    .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+    .returns("12345678") # running version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
+      .returns("") # old version
+
+    run_command("boot", config: :with_env_tags).tap do |output|
+      assert_match "docker tag dhh/app:latest dhh/app:latest", output
+      assert_match %r{docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} -e KAMAL_CONTAINER_NAME="app-web-latest" -e KAMAL_VERSION="latest" --env-file .kamal/env/roles/app-web.env --env TEST="root" --env EXPERIMENT="disabled" --env SITE="site1"}, output
+      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+    end
+  end
+
+  test "boot with web barrier opened" do
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running").at_least_once # web health check passing
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("unhealthy").at_least_once # web health check failing
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running").at_least_once # workers health check
+
+    run_command("boot", config: :with_roles, host: nil).tap do |output|
+      assert_match "Waiting for the first healthy web container before booting workers on 1.1.1.3...", output
+      assert_match "Waiting for the first healthy web container before booting workers on 1.1.1.4...", output
+      assert_match "First web container is healthy, booting workers on 1.1.1.3", output
+      assert_match "First web container is healthy, booting workers on 1.1.1.4", output
+  end
+  end
+
+  test "boot with web barrier closed" do
+    Thread.report_on_exception = false
+
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("unhealthy").at_least_once # web health check failing
+
+    stderred do
+      run_command("boot", config: :with_roles, host: nil, allow_execute_error: true).tap do |output|
+        assert_match "Waiting for the first healthy web container before booting workers on 1.1.1.3...", output
+        assert_match "Waiting for the first healthy web container before booting workers on 1.1.1.4...", output
+        assert_match "First web container is unhealthy, not booting workers on 1.1.1.3", output
+        assert_match "First web container is unhealthy, not booting workers on 1.1.1.4", output
+        assert_match "Running docker container ls --all --filter name=^app-web-latest$ --quiet | xargs docker stop on 1.1.1.1", output
+        assert_match "Running docker container ls --all --filter name=^app-web-latest$ --quiet | xargs docker stop on 1.1.1.2", output
+      end
+    end
+  ensure
+    Thread.report_on_exception = true
+  end
+
   test "start" do
     run_command("start").tap do |output|
       assert_match "docker start app-web-999", output
@@ -100,14 +176,18 @@ class CliAppTest < CliTestCase
 
   test "stop" do
     run_command("stop").tap do |output|
-      assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop", output
+      assert_match "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop", output
     end
   end
 
   test "stale_containers" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
-      .returns("12345678\n87654321")
+      .returns("12345678\n87654321\n")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("12345678\n")
 
     run_command("stale_containers").tap do |output|
       assert_match /Detected stale container for role web with version 87654321/, output
@@ -117,7 +197,11 @@ class CliAppTest < CliTestCase
   test "stop stale_containers" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
-      .returns("12345678\n87654321")
+      .returns("12345678\n87654321\n")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("12345678\n")
 
     run_command("stale_containers", "--stop").tap do |output|
       assert_match /Stopping stale container for role web with version 87654321/, output
@@ -133,7 +217,7 @@ class CliAppTest < CliTestCase
 
   test "remove" do
     run_command("remove").tap do |output|
-      assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop")}/, output
+      assert_match /#{Regexp.escape("sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop")}/, output
       assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output
       assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output
     end
@@ -165,7 +249,7 @@ class CliAppTest < CliTestCase
 
   test "exec with reuse" do
     run_command("exec", "--reuse", "ruby -v").tap do |output|
-      assert_match "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done", output # Get current version
+      assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output # Get current version
       assert_match "docker exec app-web-999 ruby -v", output
     end
   end
@@ -184,7 +268,7 @@ class CliAppTest < CliTestCase
       .with("ssh -t root@1.1.1.1 -p 22 'docker exec -it app-web-999 ruby -v'")
     run_command("exec", "-i", "--reuse", "ruby -v").tap do |output|
       assert_match "Get current version of running container...", output
-      assert_match "Running docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done on 1.1.1.1", output
+      assert_match "Running /usr/bin/env sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done on 1.1.1.1", output
       assert_match "Launching interactive command with version 999 via SSH from existing container on 1.1.1.1...", output
     end
   end
@@ -203,34 +287,58 @@ class CliAppTest < CliTestCase
 
   test "logs" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest| xargs docker logs --timestamps --tail 10 2>&1'")
+      .with("ssh -t root@1.1.1.1 'sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1| xargs docker logs --timestamps --tail 10 2>&1'")
 
-    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --tail 100 2>&1", run_command("logs")
+    assert_match "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --tail 100 2>&1", run_command("logs")
   end
 
   test "logs with follow" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 -p 22 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
+      .with("ssh -t root@1.1.1.1 -p 22 'sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
 
-    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
+    assert_match "sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
   end
 
   test "version" do
     run_command("version").tap do |output|
-      assert_match "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done", output
+      assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output
     end
   end
 
 
   test "version through main" do
-    stdouted { Kamal::Cli::Main.start(["app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1"]) }.tap do |output|
+    stdouted { Kamal::Cli::Main.start([ "app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1" ]) }.tap do |output|
-      assert_match "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done", output
+      assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output
+    end
+  end
+
+  test "long hostname" do
+    stub_running
+
+    hostname = "this-hostname-is-really-unacceptably-long-to-be-honest.example.com"
+
+    stdouted { Kamal::Cli::App.start([ "boot", "-c", "test/fixtures/deploy_with_uncommon_hostnames.yml", "--hosts", hostname ]) }.tap do |output|
+      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname this-hostname-is-really-unacceptably-long-to-be-hon-[0-9a-f]{12} /, output
+    end
+  end
+
+  test "hostname is trimmed if will end with a period" do
+    stub_running
+
+    hostname = "this-hostname-with-random-part-is-too-long.example.com"
+
+    stdouted { Kamal::Cli::App.start([ "boot", "-c", "test/fixtures/deploy_with_uncommon_hostnames.yml", "--hosts", hostname ]) }.tap do |output|
+      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname this-hostname-with-random-part-is-too-long.example-[0-9a-f]{12} /, output
     end
   end
 
   private
-    def run_command(*command, config: :with_accessories)
+    def run_command(*command, config: :with_accessories, host: "1.1.1.1", allow_execute_error: false)
-      stdouted { Kamal::Cli::App.start([*command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1"]) }
+      stdouted do
+        Kamal::Cli::App.start([ *command, "-c", "test/fixtures/deploy_#{config}.yml", *([ "--hosts", host ] if host) ])
+      rescue SSHKit::Runner::ExecuteError => e
+        raise e unless allow_execute_error
+      end
     end
 
     def stub_running

test/cli/build_test.rb

@@ -9,29 +9,137 @@ class CliBuildTest < CliTestCase
   end
 
   test "push" do
+    with_build_directory do |build_directory|
+      Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+      hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" }
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :"rev-parse", :HEAD)
+        .returns(Kamal::Git.revision)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :status, "--porcelain")
+        .returns("")
+
+      run_command("push", "--verbose").tap do |output|
+        assert_hook_ran "pre-build", output, **hook_variables
+        assert_match /Cloning repo into build directory/, output
+        assert_match /git -C #{Dir.tmpdir}\/kamal-clones\/app-#{pwd_sha} clone #{Dir.pwd}/, output
+        assert_match /docker --version && docker buildx version/, output
+        assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
+      end
+    end
+  end
+
+  test "push resetting clone" do
+    with_build_directory do |build_directory|
+      stub_setup
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd)
+        .raises(SSHKit::Command::Failed.new("fatal: destination path 'kamal' already exists and is not an empty directory"))
+        .then
+        .returns(true)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :remote, "set-url", :origin, Dir.pwd)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :fetch, :origin)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :reset, "--hard", Kamal::Git.revision)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :clean, "-fdx")
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with(:docker, :buildx, :build, "--push", "--platform", "linux/amd64,linux/arm64", "--builder", "kamal-app-multiarch", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".")
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :"rev-parse", :HEAD)
+        .returns(Kamal::Git.revision)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :status, "--porcelain")
+        .returns("")
+
+      run_command("push", "--verbose").tap do |output|
+        assert_match /Cloning repo into build directory/, output
+        assert_match /Resetting local clone/, output
+      end
+    end
+  end
+
+  test "push without clone" do
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" }
 
-    run_command("push").tap do |output|
+    run_command("push", "--verbose", fixture: :without_clone).tap do |output|
+      assert_no_match /Cloning repo into build directory/, output
       assert_hook_ran "pre-build", output, **hook_variables
       assert_match /docker --version && docker buildx version/, output
-      assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
+      assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile . as .*@localhost/, output
+    end
+  end
+
+  test "push with corrupt clone" do
+    with_build_directory do |build_directory|
+      stub_setup
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd)
+        .raises(SSHKit::Command::Failed.new("fatal: destination path 'kamal' already exists and is not an empty directory"))
+        .then
+        .returns(true)
+        .twice
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :remote, "set-url", :origin, Dir.pwd)
+        .raises(SSHKit::Command::Failed.new("fatal: not a git repository"))
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :"rev-parse", :HEAD)
+        .returns(Kamal::Git.revision)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :status, "--porcelain")
+        .returns("")
+
+      Dir.stubs(:chdir)
+
+      run_command("push", "--verbose") do |output|
+        assert_match /Cloning repo into build directory `#{build_directory}`\.\.\..*Cloning repo into build directory `#{build_directory}`\.\.\./, output
+        assert_match "Resetting local clone as `#{build_directory}` already exists...", output
+        assert_match "Error preparing clone: Failed to clone repo: fatal: not a git repository, deleting and retrying...", output
+      end
     end
   end
 
   test "push without builder" do
-    stub_setup
+    with_build_directory do |build_directory|
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      stub_setup
-      .with(:docker, "--version", "&&", :docker, :buildx, "version")
 
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
-      .with { |*args| args[0..1] == [:docker, :buildx] }
+        .with(:docker, "--version", "&&", :docker, :buildx, "version")
-      .raises(SSHKit::Command::Failed.new("no builder"))
-      .then
-      .returns(true)
 
-    run_command("push").tap do |output|
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
-      assert_match /Missing compatible builder, so creating a new one first/, output
+        .with(:docker, :buildx, :create, "--use", "--name", "kamal-app-multiarch")
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with { |*args| args[0..1] == [ :docker, :buildx ] }
+        .raises(SSHKit::Command::Failed.new("no builder"))
+        .then
+        .returns(true)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with { |*args| args.first.start_with?("git") }
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :"rev-parse", :HEAD)
+        .returns(Kamal::Git.revision)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :status, "--porcelain")
+        .returns("")
+
+      run_command("push").tap do |output|
+        assert_match /WARN Missing compatible builder, so creating a new one first/, output
+      end
     end
   end
 
@@ -48,9 +156,10 @@ class CliBuildTest < CliTestCase
   test "push pre-build hook failure" do
     fail_hook("pre-build")
 
-    assert_raises(Kamal::Cli::HookError) { run_command("push") }
+    error = assert_raises(Kamal::Cli::HookError) { run_command("push") }
+    assert_equal "Hook `pre-build` failed:\nfailed", error.message
 
-    assert @executions.none? { |args| args[0..2] == [:docker, :buildx, :build] }
+    assert @executions.none? { |args| args[0..2] == [ :docker, :buildx, :build ] }
   end
 
   test "pull" do
@@ -75,6 +184,14 @@ class CliBuildTest < CliTestCase
     end
   end
 
+  test "create remote with custom ports" do
+    run_command("create", fixture: :with_remote_builder_and_custom_ports).tap do |output|
+      assert_match "Running /usr/bin/env true on 1.1.1.5", output
+      assert_match "docker context create kamal-app-native-remote-amd64 --description 'kamal-app-native-remote amd64 native host' --docker 'host=ssh://app@1.1.1.5:2122'", output
+      assert_match "docker buildx create --name kamal-app-native-remote kamal-app-native-remote-amd64 --platform linux/amd64", output
+    end
+  end
+
   test "create with error" do
     stub_setup
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
@@ -105,13 +222,26 @@ class CliBuildTest < CliTestCase
 
   private
     def run_command(*command, fixture: :with_accessories)
-      stdouted { Kamal::Cli::Build.start([*command, "-c", "test/fixtures/deploy_#{fixture}.yml"]) }
+      stdouted { Kamal::Cli::Build.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) }
     end
 
     def stub_dependency_checks
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
         .with(:docker, "--version", "&&", :docker, :buildx, "version")
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |*args| args[0..1] == [:docker, :buildx] }
+        .with { |*args| args[0..1] == [ :docker, :buildx ] }
+    end
+
+    def with_build_directory
+      build_directory = File.join Dir.tmpdir, "kamal-clones", "app-#{pwd_sha}", "kamal"
+      FileUtils.mkdir_p build_directory
+      FileUtils.touch File.join build_directory, "Dockerfile"
+      yield build_directory + "/"
+    ensure
+      FileUtils.rm_rf build_directory
+    end
+
+    def pwd_sha
+      Digest::SHA256.hexdigest(Dir.pwd)[0..12]
     end
 end

test/cli/cli_test_case.rb

@@ -21,7 +21,7 @@ class CliTestCase < ActiveSupport::TestCase
       Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
 
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |*args| @executions << args; args != [".kamal/hooks/#{hook}"] }
+        .with { |*args| @executions << args; args != [ ".kamal/hooks/#{hook}" ] }
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
         .with { |*args| args.first == ".kamal/hooks/#{hook}" }
         .raises(SSHKit::Command::Failed.new("failed"))
@@ -31,12 +31,14 @@ class CliTestCase < ActiveSupport::TestCase
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
         .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :mkdir && arg2 == ".kamal/lock-app" }
+        .with { |arg1, arg2, arg3| arg1 == :mkdir && arg2 == "-p" && arg3 == ".kamal/locks" }
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :rm && arg2 == ".kamal/lock-app/details" }
+        .with { |arg1, arg2| arg1 == :mkdir && arg2 == ".kamal/locks/app" }
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |arg1, arg2| arg1 == :rm && arg2 == ".kamal/locks/app/details" }
     end
 
-    def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: nil)
+    def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: false)
       performer = `whoami`.strip
 
       assert_match "Running the #{hook} hook...\n", output
@@ -50,7 +52,7 @@ class CliTestCase < ActiveSupport::TestCase
         KAMAL_HOSTS=\"#{hosts}\"\s
         KAMAL_COMMAND=\"#{command}\"\s
         #{"KAMAL_SUBCOMMAND=\\\"#{subcommand}\\\"\\s" if subcommand}
-        #{"KAMAL_RUNTIME=\\\"#{runtime}\\\"\\s" if runtime}
+        #{"KAMAL_RUNTIME=\\\"\\d+\\\"\\s" if runtime}
         ;\s/usr/bin/env\s\.kamal/hooks/#{hook} }x
 
       assert_match expected, output

test/cli/env_test.rb

@@ -13,7 +13,6 @@ class CliEnvTest < CliTestCase
       assert_match ".kamal/env/roles/app-workers.env", output
       assert_match ".kamal/env/traefik/traefik.env", output
       assert_match ".kamal/env/accessories/app-redis.env", output
-
     end
   end
 
@@ -33,6 +32,6 @@ class CliEnvTest < CliTestCase
 
   private
     def run_command(*command)
-      stdouted { Kamal::Cli::Env.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Env.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
     end
 end

test/cli/healthcheck_test.rb

@@ -1,82 +0,0 @@
-require_relative "cli_test_case"
-
-class CliHealthcheckTest < CliTestCase
-  test "perform" do
-    # Prevent expected failures from outputting to terminal
-    Thread.report_on_exception = false
-
-    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
-    Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
-
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--env-file", ".kamal/env/roles/app-web.env", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false)
-
-    # Fail twice to test retry logic
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-      .returns("starting")
-      .then
-      .returns("unhealthy")
-      .then
-      .returns("healthy")
-
-    run_command("perform").tap do |output|
-      assert_match "container not ready (starting), retrying in 1s (attempt 1/7)...", output
-      assert_match "container not ready (unhealthy), retrying in 2s (attempt 2/7)...", output
-      assert_match "Container is healthy!", output
-    end
-  end
-
-  test "perform failing to become healthy" do
-    # Prevent expected failures from outputting to terminal
-    Thread.report_on_exception = false
-
-    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
-
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--env-file", ".kamal/env/roles/app-web.env", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false)
-
-    # Continually report unhealthy
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-      .returns("unhealthy")
-
-    # Capture logs when failing
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :logs, "--tail", 50, "2>&1")
-      .returns("some log output")
-
-    # Capture container health log when failing
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_pretty_json)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :inspect, "--format",  "'{{json .State.Health}}'")
-      .returns('{"Status":"unhealthy","Log":[{"ExitCode": 1,"Output": "/bin/sh: 1: curl: not found\n"}]}"')
-
-    exception = assert_raises do
-      run_command("perform")
-    end
-    assert_match "container not ready (unhealthy)", exception.message
-  end
-  
-  test "raises an exception if primary does not have traefik" do
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).never
-    
-    exception = assert_raises do
-      run_command("perform", config_file: "test/fixtures/deploy_workers_only.yml")
-    end
-
-    assert_equal "The primary host is not configured to run Traefik", exception.message
-  end
-
-  private
-    def run_command(*command, config_file: "test/fixtures/deploy_with_accessories.yml")
-      stdouted { Kamal::Cli::Healthcheck.start([*command, "-c", config_file]) } 
-    end
-end

test/cli/lock_test.rb

@@ -3,7 +3,7 @@ require_relative "cli_test_case"
 class CliLockTest < CliTestCase
   test "status" do
     run_command("status").tap do |output|
-      assert_match "Running /usr/bin/env stat .kamal/lock-app > /dev/null && cat .kamal/lock-app/details | base64 -d on 1.1.1.1", output
+      assert_match "Running /usr/bin/env stat .kamal/locks/app > /dev/null && cat .kamal/locks/app/details | base64 -d on 1.1.1.1", output
     end
   end
 
@@ -15,6 +15,6 @@ class CliLockTest < CliTestCase
 
   private
     def run_command(*command)
-      stdouted { Kamal::Cli::Lock.start([*command, "-v", "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Lock.start([ *command, "-v", "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
     end
 end

test/cli/main_test.rb

@@ -5,13 +5,13 @@ class CliMainTest < CliTestCase
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:main:envify", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:deploy)
 
     run_command("setup").tap do |output|
       assert_match /Ensure Docker is installed.../, output
-      assert_match /Push env files.../, output
+      assert_match /Evaluate and push env files.../, output
     end
   end
 
@@ -19,26 +19,24 @@ class CliMainTest < CliTestCase
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:main:envify", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
     # deploy
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
     run_command("setup", "--skip_push").tap do |output|
       assert_match /Ensure Docker is installed.../, output
-      assert_match /Push env files.../, output
+      assert_match /Evaluate and push env files.../, output
       # deploy
       assert_match /Acquiring the deploy lock/, output
       assert_match /Log into image registry/, output
       assert_match /Pull app image/, output
       assert_match /Ensure Traefik is running/, output
-      assert_match /Ensure app can pass healthcheck/, output
       assert_match /Detect stale containers/, output
       assert_match /Prune old containers and images/, output
       assert_match /Releasing the deploy lock/, output
@@ -46,12 +44,11 @@ class CliMainTest < CliTestCase
   end
 
   test "deploy" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
@@ -59,16 +56,15 @@ class CliMainTest < CliTestCase
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "deploy" }
 
-    run_command("deploy").tap do |output|
+    run_command("deploy", "--verbose").tap do |output|
       assert_hook_ran "pre-connect", output, **hook_variables
       assert_match /Log into image registry/, output
       assert_match /Build and push app image/, output
       assert_hook_ran "pre-deploy", output, **hook_variables
       assert_match /Ensure Traefik is running/, output
-      assert_match /Ensure app can pass healthcheck/, output
       assert_match /Detect stale containers/, output
       assert_match /Prune old containers and images/, output
-      assert_hook_ran "post-deploy", output, **hook_variables, runtime: 0
+      assert_hook_ran "post-deploy", output, **hook_variables, runtime: true
     end
   end
 
@@ -78,7 +74,6 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
@@ -88,7 +83,6 @@ class CliMainTest < CliTestCase
       assert_match /Log into image registry/, output
       assert_match /Pull app image/, output
       assert_match /Ensure Traefik is running/, output
-      assert_match /Ensure app can pass healthcheck/, output
       assert_match /Detect stale containers/, output
       assert_match /Prune old containers and images/, output
       assert_match /Releasing the deploy lock/, output
@@ -98,15 +92,29 @@ class CliMainTest < CliTestCase
   test "deploy when locked" do
     Thread.report_on_exception = false
 
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+    Dir.stubs(:chdir)
+
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*arg| arg[0..1] == [:mkdir, ".kamal/lock-app"] }
+      .with { |*args| args == [ :mkdir, "-p", ".kamal/locks" ] }
-      .raises(RuntimeError, "mkdir: cannot create directory ‘kamal_lock-app’: File exists")
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |*arg| arg[0..1] == [ :mkdir, ".kamal/locks/app" ] }
+      .raises(RuntimeError, "mkdir: cannot create directory ‘kamal/locks/app’: File exists")
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_debug)
-      .with(:stat, ".kamal/lock-app", ">", "/dev/null", "&&", :cat, ".kamal/lock-app/details", "|", :base64, "-d")
+      .with(:stat, ".kamal/locks/app", ">", "/dev/null", "&&", :cat, ".kamal/locks/app/details", "|", :base64, "-d")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:git, "-C", anything, :"rev-parse", :HEAD)
+      .returns(Kamal::Git.revision)
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:git, "-C", anything, :status, "--porcelain")
+      .returns("")
 
     assert_raises(Kamal::Cli::LockError) do
       run_command("deploy")
@@ -116,13 +124,27 @@ class CliMainTest < CliTestCase
   test "deploy error when locking" do
     Thread.report_on_exception = false
 
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+    Dir.stubs(:chdir)
+
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*arg| arg[0..1] == [:mkdir, ".kamal/lock-app"] }
+      .with { |*args| args == [ :mkdir, "-p", ".kamal/locks" ] }
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |*arg| arg[0..1] == [ :mkdir, ".kamal/locks/app" ] }
       .raises(SocketError, "getaddrinfo: nodename nor servname provided, or not known")
 
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:git, "-C", anything, :"rev-parse", :HEAD)
+      .returns(Kamal::Git.revision)
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:git, "-C", anything, :status, "--porcelain")
+      .returns("")
+
     assert_raises(SSHKit::Runner::ExecuteError) do
       run_command("deploy")
     end
@@ -135,11 +157,11 @@ class CliMainTest < CliTestCase
       .with("kamal:cli:registry:login", [], invoke_options)
       .raises(RuntimeError)
 
-    assert !KAMAL.holding_lock?
+    assert_not KAMAL.holding_lock?
     assert_raises(RuntimeError) do
       stderred { run_command("deploy") }
     end
-    assert !KAMAL.holding_lock?
+    assert_not KAMAL.holding_lock?
   end
 
   test "deploy with skipped hooks" do
@@ -148,13 +170,12 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
     run_command("deploy", "--skip_hooks") do
-      refute_match /Running the post-deploy hook.../, output
+      assert_no_match /Running the post-deploy hook.../, output
     end
   end
 
@@ -179,7 +200,6 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
@@ -188,10 +208,9 @@ class CliMainTest < CliTestCase
   end
 
   test "redeploy" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
 
@@ -199,13 +218,12 @@ class CliMainTest < CliTestCase
 
     hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "redeploy" }
 
-    run_command("redeploy").tap do |output|
+    run_command("redeploy", "--verbose").tap do |output|
       assert_hook_ran "pre-connect", output, **hook_variables
       assert_match /Build and push app image/, output
       assert_hook_ran "pre-deploy", output, **hook_variables
       assert_match /Running the pre-deploy hook.../, output
-      assert_match /Ensure app can pass healthcheck/, output
+      assert_hook_ran "post-deploy", output, **hook_variables, runtime: true
-      assert_hook_ran "post-deploy", output, **hook_variables, runtime: "0"
     end
   end
 
@@ -213,13 +231,11 @@ class CliMainTest < CliTestCase
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
 
     run_command("redeploy", "--skip_push").tap do |output|
       assert_match /Pull app image/, output
-      assert_match /Ensure app can pass healthcheck/, output
     end
   end
 
@@ -244,7 +260,7 @@ class CliMainTest < CliTestCase
         .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet")
         .returns("version-to-rollback\n").at_least_once
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-#{role}-}; done", raise_on_non_zero_exit: false)
+        .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=#{role} --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=#{role} --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-#{role}-}; done", raise_on_non_zero_exit: false)
         .returns("version-to-rollback\n").at_least_once
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
         .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
@@ -252,7 +268,7 @@ class CliMainTest < CliTestCase
     end
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
       .returns("corddirectory").at_least_once # health check
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -262,12 +278,12 @@ class CliMainTest < CliTestCase
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     hook_variables = { version: 123, service_version: "app@123", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "rollback" }
 
-    run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
+    run_command("rollback", "--verbose", "123", config_file: "deploy_with_accessories").tap do |output|
       assert_hook_ran "pre-deploy", output, **hook_variables
       assert_match "docker tag dhh/app:123 dhh/app:latest", output
       assert_match "docker run --detach --restart unless-stopped --name app-web-123", output
       assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running"
-      assert_hook_ran "post-deploy", output, **hook_variables, runtime: "0"
+      assert_hook_ran "post-deploy", output, **hook_variables, runtime: true
     end
   end
 
@@ -280,7 +296,7 @@ class CliMainTest < CliTestCase
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
       .returns("").at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
       .returns("").at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
@@ -311,8 +327,8 @@ class CliMainTest < CliTestCase
     run_command("config", config_file: "deploy_simple").tap do |output|
       config = YAML.load(output)
 
-      assert_equal ["web"], config[:roles]
+      assert_equal [ "web" ], config[:roles]
-      assert_equal ["1.1.1.1", "1.1.1.2"], config[:hosts]
+      assert_equal [ "1.1.1.1", "1.1.1.2" ], config[:hosts]
       assert_equal "999", config[:version]
       assert_equal "dhh/app", config[:repository]
       assert_equal "dhh/app:999", config[:absolute_image]
@@ -324,8 +340,8 @@ class CliMainTest < CliTestCase
     run_command("config", config_file: "deploy_with_roles").tap do |output|
       config = YAML.load(output)
 
-      assert_equal ["web", "workers"], config[:roles]
+      assert_equal [ "web", "workers" ], config[:roles]
-      assert_equal ["1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4"], config[:hosts]
+      assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], config[:hosts]
       assert_equal "999", config[:version]
       assert_equal "registry.digitalocean.com/dhh/app", config[:repository]
       assert_equal "registry.digitalocean.com/dhh/app:999", config[:absolute_image]
@@ -337,8 +353,8 @@ class CliMainTest < CliTestCase
     run_command("config", config_file: "deploy_primary_web_role_override").tap do |output|
       config = YAML.load(output)
 
-      assert_equal ["web_chicago", "web_tokyo"], config[:roles]
+      assert_equal [ "web_chicago", "web_tokyo" ], config[:roles]
-      assert_equal ["1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4"], config[:hosts]
+      assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], config[:hosts]
       assert_equal "1.1.1.3", config[:primary_host]
     end
   end
@@ -347,8 +363,8 @@ class CliMainTest < CliTestCase
     run_command("config", "-d", "world", config_file: "deploy_for_dest").tap do |output|
       config = YAML.load(output)
 
-      assert_equal ["web"], config[:roles]
+      assert_equal [ "web" ], config[:roles]
-      assert_equal ["1.1.1.1", "1.1.1.2"], config[:hosts]
+      assert_equal [ "1.1.1.1", "1.1.1.2" ], config[:hosts]
       assert_equal "999", config[:version]
       assert_equal "registry.digitalocean.com/dhh/app", config[:repository]
       assert_equal "registry.digitalocean.com/dhh/app:999", config[:absolute_image]
@@ -360,8 +376,8 @@ class CliMainTest < CliTestCase
     run_command("config", config_file: "deploy_with_aliases").tap do |output|
       config = YAML.load(output)
 
-      assert_equal ["web", "web_tokyo", "workers", "workers_tokyo"], config[:roles]
+      assert_equal [ "web", "web_tokyo", "workers", "workers_tokyo" ], config[:roles]
-      assert_equal ["1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4"], config[:hosts]
+      assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], config[:hosts]
       assert_equal "999", config[:version]
       assert_equal "registry.digitalocean.com/dhh/app", config[:repository]
       assert_equal "registry.digitalocean.com/dhh/app:999", config[:absolute_image]
@@ -421,6 +437,7 @@ class CliMainTest < CliTestCase
   end
 
   test "envify" do
+    Pathname.any_instance.expects(:exist?).returns(true).times(3)
     File.expects(:read).with(".env.erb").returns("HELLO=<%= 'world' %>")
     File.expects(:write).with(".env", "HELLO=world", perm: 0600)
 
@@ -435,6 +452,7 @@ class CliMainTest < CliTestCase
       <% end  -%>
     EOF
 
+    Pathname.any_instance.expects(:exist?).returns(true).times(3)
     File.expects(:read).with(".env.erb").returns(file.strip)
     File.expects(:write).with(".env", "HELLO=world\nKEY=value\n", perm: 0600)
 
@@ -442,6 +460,7 @@ class CliMainTest < CliTestCase
   end
 
   test "envify with destination" do
+    Pathname.any_instance.expects(:exist?).returns(true).times(4)
     File.expects(:read).with(".env.world.erb").returns("HELLO=<%= 'world' %>")
     File.expects(:write).with(".env.world", "HELLO=world", perm: 0600)
 
@@ -449,6 +468,7 @@ class CliMainTest < CliTestCase
   end
 
   test "envify with skip_push" do
+    Pathname.any_instance.expects(:exist?).returns(true).times(1)
     File.expects(:read).with(".env.erb").returns("HELLO=<%= 'world' %>")
     File.expects(:write).with(".env", "HELLO=world", perm: 0600)
 
@@ -487,6 +507,6 @@ class CliMainTest < CliTestCase
 
   private
     def run_command(*command, config_file: "deploy_simple")
-      stdouted { Kamal::Cli::Main.start([*command, "-c", "test/fixtures/#{config_file}.yml"]) }
+      stdouted { Kamal::Cli::Main.start([ *command, "-c", "test/fixtures/#{config_file}.yml" ]) }
     end
 end

test/cli/prune_test.rb

@@ -33,6 +33,6 @@ class CliPruneTest < CliTestCase
 
   private
     def run_command(*command)
-      stdouted { Kamal::Cli::Prune.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Prune.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
     end
 end

test/cli/registry_test.rb

@@ -16,6 +16,6 @@ class CliRegistryTest < CliTestCase
 
   private
     def run_command(*command)
-      stdouted { Kamal::Cli::Registry.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Registry.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
     end
 end

test/cli/server_test.rb

@@ -1,14 +1,30 @@
 require_relative "cli_test_case"
 
 class CliServerTest < CliTestCase
+  test "running a command with exec" do
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture)
+    .with("date", verbosity: 1)
+    .returns("Today")
+
+    hosts = "1.1.1.1".."1.1.1.4"
+    run_command("exec", "date").tap do |output|
+      hosts.map do |host|
+        assert_match "Running 'date' on #{hosts.to_a.join(', ')}...", output
+        assert_match "App Host: #{host}\nToday", output
+      end
+    end
+  end
+
   test "bootstrap already installed" do
+    stub_setup
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(true).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
 
-    assert_equal "", run_command("bootstrap")
+    assert_equal "Acquiring the deploy lock...\nReleasing the deploy lock...", run_command("bootstrap")
   end
 
   test "bootstrap install as non-root user" do
+    stub_setup
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', raise_on_non_zero_exit: false).returns(false).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
@@ -19,11 +35,13 @@ class CliServerTest < CliTestCase
   end
 
   test "bootstrap install as root user" do
+    stub_setup
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', raise_on_non_zero_exit: false).returns(true).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:sh, "-c", "'curl -fsSL https://get.docker.com || wget -O - https://get.docker.com || echo \"exit 1\"'", "|", :sh).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(".kamal/hooks/pre-connect", anything).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(".kamal/hooks/docker-setup", anything).at_least_once
 
     run_command("bootstrap").tap do |output|
@@ -36,6 +54,6 @@ class CliServerTest < CliTestCase
 
   private
     def run_command(*command)
-      stdouted { Kamal::Cli::Server.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Server.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
     end
 end

test/cli/traefik_test.rb

@@ -11,7 +11,7 @@ class CliTraefikTest < CliTestCase
   test "reboot" do
     Kamal::Commands::Registry.any_instance.expects(:login).twice
 
-    run_command("reboot").tap do |output|
+    run_command("reboot", "-y").tap do |output|
       assert_match "docker container stop traefik", output
       assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik", output
       assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
@@ -21,7 +21,7 @@ class CliTraefikTest < CliTestCase
   test "reboot --rolling" do
     Object.any_instance.stubs(:sleep)
 
-    run_command("reboot", "--rolling").tap do |output|
+    run_command("reboot", "--rolling", "-y").tap do |output|
       assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.1", output
     end
   end
@@ -91,6 +91,6 @@ class CliTraefikTest < CliTestCase
 
   private
     def run_command(*command)
-      stdouted { Kamal::Cli::Traefik.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Traefik.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
     end
 end

test/commander_test.rb

@@ -24,6 +24,9 @@ class CommanderTest < ActiveSupport::TestCase
     @kamal.specific_hosts = [ "*" ]
     assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @kamal.hosts
 
+    @kamal.specific_hosts = [ "1.1.1.[12]" ]
+    assert_equal [ "1.1.1.1", "1.1.1.2" ], @kamal.hosts
+
     exception = assert_raises(ArgumentError) do
       @kamal.specific_hosts = [ "*miss" ]
     end
@@ -57,6 +60,9 @@ class CommanderTest < ActiveSupport::TestCase
     @kamal.specific_roles = [ "*" ]
     assert_equal [ "web", "workers" ], @kamal.roles.map(&:name)
 
+    @kamal.specific_roles = [ "w{eb,orkers}" ]
+    assert_equal [ "web", "workers" ], @kamal.roles.map(&:name)
+
     exception = assert_raises(ArgumentError) do
       @kamal.specific_roles = [ "*miss" ]
     end
@@ -93,6 +99,11 @@ class CommanderTest < ActiveSupport::TestCase
     assert_equal [ "workers" ], @kamal.roles_on("1.1.1.3").map(&:name)
   end
 
+  test "roles_on web comes first" do
+    configure_with(:deploy_with_two_roles_one_host)
+    assert_equal [ "web", "workers" ], @kamal.roles_on("1.1.1.1").map(&:name)
+  end
+
   test "default group strategy" do
     assert_empty @kamal.boot_strategy
   end
@@ -119,9 +130,24 @@ class CommanderTest < ActiveSupport::TestCase
     configure_with(:deploy_primary_web_role_override)
 
     @kamal.specific_roles = [ "web_*" ]
-    assert_equal [ "web_chicago", "web_tokyo" ], @kamal.roles.map(&:name)
+    assert_equal [ "web_tokyo", "web_chicago" ], @kamal.roles.map(&:name)
     assert_equal "web_tokyo", @kamal.primary_role.name
     assert_equal "1.1.1.3", @kamal.primary_host
+    assert_equal [ "1.1.1.3", "1.1.1.4", "1.1.1.1", "1.1.1.2" ], @kamal.hosts
+  end
+
+  test "traefik hosts should observe filtered roles" do
+    configure_with(:deploy_with_aliases)
+
+    @kamal.specific_roles = [ "web_tokyo" ]
+    assert_equal [ "1.1.1.3", "1.1.1.4" ], @kamal.traefik_hosts
+  end
+
+  test "traefik hosts should observe filtered hosts" do
+    configure_with(:deploy_with_aliases)
+
+    @kamal.specific_hosts = [ "1.1.1.4" ]
+    assert_equal [ "1.1.1.4" ], @kamal.traefik_hosts
   end
 
   private

test/commands/accessory_test.rb

@@ -50,11 +50,11 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --label service=\"app-mysql\" private.registry/mysql:8.0",
+      "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --label service=\"app-mysql\" private.registry/mysql:8.0",
       new_command(:mysql).run.join(" ")
 
     assert_equal \
-      "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest",
+      "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --env SOMETHING=\"else\" --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest",
       new_command(:redis).run.join(" ")
 
     assert_equal \
@@ -91,7 +91,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "execute in new container" do
     assert_equal \
-      "docker run --rm --env-file .kamal/env/accessories/app-mysql.env private.registry/mysql:8.0 mysql -u root",
+      "docker run --rm --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" private.registry/mysql:8.0 mysql -u root",
       new_command(:mysql).execute_in_new_container("mysql", "-u", "root").join(" ")
   end
 
@@ -103,14 +103,14 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "execute in new container over ssh" do
     new_command(:mysql).stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do
-      assert_match %r|docker run -it --rm --env-file .kamal/env/accessories/app-mysql.env private.registry/mysql:8.0 mysql -u root|,
+      assert_match %r{docker run -it --rm --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" private.registry/mysql:8.0 mysql -u root},
         new_command(:mysql).execute_in_new_container_over_ssh("mysql", "-u", "root")
     end
   end
 
   test "execute in existing container over ssh" do
     new_command(:mysql).stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do
-      assert_match %r|docker exec -it app-mysql mysql -u root|,
+      assert_match %r{docker exec -it app-mysql mysql -u root},
         new_command(:mysql).execute_in_existing_container_over_ssh("mysql", "-u", "root")
     end
   end

test/commands/app_test.rb

@@ -14,21 +14,21 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with hostname" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run(hostname: "myhost").join(" ")
   end
 
   test "run with volumes" do
-    @config[:volumes] = ["/local/path:/container/path" ]
+    @config[:volumes] = [ "/local/path:/container/path" ]
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -36,7 +36,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "path" => "/healthz" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -44,7 +44,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "cmd" => "/bin/up" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -52,22 +52,22 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with custom options" do
     @config[:servers] = { "web" => [ "1.1.1.1" ], "jobs" => { "hosts" => [ "1.1.1.2" ], "cmd" => "bin/jobs", "options" => { "mount" => "somewhere", "cap-add" => true } } }
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-jobs.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
+      "docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-jobs.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --label destination --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
-      new_command(role: "jobs").run.join(" ")
+      new_command(role: "jobs", host: "1.1.1.2").run.join(" ")
   end
 
   test "run with logging config" do
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -76,7 +76,16 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "logging" => { "driver" => "local", "options" => { "max-size" => "100m" } } } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      new_command.run.join(" ")
+  end
+
+  test "run with tags" do
+    @config[:servers] = [ { "1.1.1.1" => "tag1" } ]
+    @config[:env]["tags"] = { "tag1" => { "ENV1" => "value1" } }
+
+    assert_equal \
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -95,14 +104,14 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "stop" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop",
       new_command.stop.join(" ")
   end
 
   test "stop with custom stop wait time" do
     @config[:stop_wait_time] = 30
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop -t 30",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop -t 30",
       new_command.stop.join(" ")
   end
 
@@ -128,45 +137,45 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "logs" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs 2>&1",
       new_command.logs.join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --since 5m 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --since 5m 2>&1",
       new_command.logs(since: "5m").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --tail 100 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --tail 100 2>&1",
       new_command.logs(lines: "100").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --since 5m --tail 100 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --since 5m --tail 100 2>&1",
       new_command.logs(since: "5m", lines: "100").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs 2>&1 | grep 'my-id'",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs 2>&1 | grep 'my-id'",
       new_command.logs(grep: "my-id").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
       new_command.logs(since: "5m", grep: "my-id").join(" ")
   end
 
   test "follow logs" do
-    assert_match \
+    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --follow 2>&1",
+      "ssh -t root@app-1 -p 22 'sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --follow 2>&1'",
       new_command.follow_logs(host: "app-1")
 
-    assert_match \
+    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --follow 2>&1 | grep \"Completed\"",
+      "ssh -t root@app-1 -p 22 'sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --follow 2>&1 | grep \"Completed\"'",
       new_command.follow_logs(host: "app-1", grep: "Completed")
 
-    assert_match \
+    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 123 --follow 2>&1",
+      "ssh -t root@app-1 -p 22 'sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --tail 123 --follow 2>&1'",
       new_command.follow_logs(host: "app-1", lines: 123)
 
-    assert_match \
+    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 123 --follow 2>&1 | grep \"Completed\"",
+      "ssh -t root@app-1 -p 22 'sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --tail 123 --follow 2>&1 | grep \"Completed\"'",
       new_command.follow_logs(host: "app-1", lines: 123, grep: "Completed")
   end
 
@@ -174,36 +183,65 @@ class CommandsAppTest < ActiveSupport::TestCase
   test "execute in new container" do
     assert_equal \
       "docker run --rm --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails db:setup",
-      new_command.execute_in_new_container("bin/rails", "db:setup").join(" ")
+      new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ")
+  end
+
+  test "execute in new container with env" do
+    assert_equal \
+      "docker run --rm --env-file .kamal/env/roles/app-web.env --env foo=\"bar\" dhh/app:999 bin/rails db:setup",
+      new_command.execute_in_new_container("bin/rails", "db:setup", env: { "foo" => "bar" }).join(" ")
+  end
+
+  test "execute in new container with tags" do
+    @config[:servers] = [ { "1.1.1.1" => "tag1" } ]
+    @config[:env]["tags"] = { "tag1" => { "ENV1" => "value1" } }
+
+    assert_equal \
+      "docker run --rm --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" dhh/app:999 bin/rails db:setup",
+      new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ")
   end
 
   test "execute in new container with custom options" do
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } }
     assert_equal \
       "docker run --rm --env-file .kamal/env/roles/app-web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails db:setup",
-      new_command.execute_in_new_container("bin/rails", "db:setup").join(" ")
+      new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ")
   end
 
   test "execute in existing container" do
     assert_equal \
       "docker exec app-web-999 bin/rails db:setup",
-      new_command.execute_in_existing_container("bin/rails", "db:setup").join(" ")
+      new_command.execute_in_existing_container("bin/rails", "db:setup", env: {}).join(" ")
+  end
+
+  test "execute in existing container with env" do
+    assert_equal \
+      "docker exec --env foo=\"bar\" app-web-999 bin/rails db:setup",
+      new_command.execute_in_existing_container("bin/rails", "db:setup", env: { "foo" => "bar" }).join(" ")
   end
 
   test "execute in new container over ssh" do
-    assert_match %r|docker run -it --rm --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails c|,
+    assert_match %r{docker run -it --rm --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails c},
-      new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1")
+      new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {})
+  end
+
+  test "execute in new container over ssh with tags" do
+    @config[:servers] = [ { "1.1.1.1" => "tag1" } ]
+    @config[:env]["tags"] = { "tag1" => { "ENV1" => "value1" } }
+
+    assert_equal "ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" dhh/app:999 bin/rails c'",
+      new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {})
   end
 
   test "execute in new container with custom options over ssh" do
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } }
-    assert_match %r|docker run -it --rm --env-file .kamal/env/roles/app-web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails c|,
+    assert_match %r{docker run -it --rm --env-file .kamal/env/roles/app-web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails c},
-      new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1")
+      new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {})
   end
 
   test "execute in existing container over ssh" do
-    assert_match %r|docker exec -it app-web-999 bin/rails c|,
+    assert_match %r{docker exec -it app-web-999 bin/rails c},
-      new_command.execute_in_existing_container_over_ssh("bin/rails", "c", host: "app-1")
+      new_command.execute_in_existing_container_over_ssh("bin/rails", "c", env: {})
   end
 
   test "run over ssh" do
@@ -242,14 +280,14 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "current_running_container_id" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest",
+    "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1",
       new_command.current_running_container_id.join(" ")
   end
 
   test "current_running_container_id with destination" do
     @destination = "staging"
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --filter status=restarting --latest",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest-staging --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --filter status=restarting' | head -1",
       new_command.current_running_container_id.join(" ")
   end
 
@@ -261,7 +299,7 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "current_running_version" do
     assert_equal \
-      "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done",
+      "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done",
       new_command.current_running_version.join(" ")
   end
 
@@ -339,10 +377,17 @@ class CommandsAppTest < ActiveSupport::TestCase
       new_command.remove_images.join(" ")
   end
 
-  test "tag_current_image_as_latest" do
+  test "tag_latest_image" do
     assert_equal \
       "docker tag dhh/app:999 dhh/app:latest",
-      new_command.tag_current_image_as_latest.join(" ")
+      new_command.tag_latest_image.join(" ")
+  end
+
+  test "tag_latest_image with destination" do
+    @destination = "staging"
+    assert_equal \
+      "docker tag dhh/app:999 dhh/app:latest-staging",
+      new_command.tag_latest_image.join(" ")
   end
 
   test "make_env_directory" do
@@ -371,7 +416,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     assert_equal [
       :mkdir, "-p", ".kamal/assets/extracted/app-web-999", "&&",
       :docker, :stop, "-t 1", "app-web-assets", "2> /dev/null", "|| true", "&&",
-      :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "dhh/app:latest", "sleep 1000000", "&&",
+      :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "dhh/app:999", "sleep 1000000", "&&",
       :docker, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/assets/extracted/app-web-999", "&&",
       :docker, :stop, "-t 1", "app-web-assets"
     ], new_command(asset_path: "/public/assets").extract_assets
@@ -387,7 +432,7 @@ class CommandsAppTest < ActiveSupport::TestCase
       :mkdir, "-p", ".kamal/assets/volumes/app-web-999", ";",
       :cp, "-rnT", ".kamal/assets/extracted/app-web-999", ".kamal/assets/volumes/app-web-999", ";",
       :cp, "-rnT", ".kamal/assets/extracted/app-web-999", ".kamal/assets/volumes/app-web-998", "|| true", ";",
-      :cp, "-rnT", ".kamal/assets/extracted/app-web-998", ".kamal/assets/volumes/app-web-999", "|| true",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-998", ".kamal/assets/volumes/app-web-999", "|| true"
     ], new_command(asset_path: "/public/assets").sync_asset_volumes(old_version: 998)
   end
 
@@ -399,8 +444,8 @@ class CommandsAppTest < ActiveSupport::TestCase
   end
 
   private
-    def new_command(role: "web", **additional_config)
+    def new_command(role: "web", host: "1.1.1.1", **additional_config)
       config = Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999")
-      Kamal::Commands::App.new(config, role: config.role(role))
+      Kamal::Commands::App.new(config, role: config.role(role), host: host)
     end
 end

test/commands/builder_test.rb

@@ -6,7 +6,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
   end
 
   test "target multiarch by default" do
-    builder = new_builder_command(builder: { "cache" => { "type" => "gha" }})
+    builder = new_builder_command(builder: { "cache" => { "type" => "gha" } })
     assert_equal "multiarch", builder.name
     assert_equal \
       "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
@@ -22,7 +22,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
   end
 
   test "target native cached when multiarch is off and cache is set" do
-    builder = new_builder_command(builder: { "multiarch" => false, "cache" => { "type" => "gha" }})
+    builder = new_builder_command(builder: { "multiarch" => false, "cache" => { "type" => "gha" } })
     assert_equal "native/cached", builder.name
     assert_equal \
       "docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
@@ -30,7 +30,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
   end
 
   test "target multiarch remote when local and remote is set" do
-    builder = new_builder_command(builder: { "local" => { }, "remote" => { }, "cache" => { "type" => "gha" } })
+    builder = new_builder_command(builder: { "local" => {}, "remote" => {}, "cache" => { "type" => "gha" } })
     assert_equal "multiarch/remote", builder.name
     assert_equal \
       "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
@@ -61,7 +61,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
   end
 
   test "build secrets" do
-    builder = new_builder_command(builder: { "secrets" => ["token_a", "token_b"] })
+    builder = new_builder_command(builder: { "secrets" => [ "token_a", "token_b" ] })
     assert_equal \
       "-t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"token_a\" --secret id=\"token_b\" --file Dockerfile",
       builder.target.build_options.join(" ")
@@ -83,6 +83,13 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     end
   end
 
+  test "build target" do
+    builder = new_builder_command(builder: { "target" => "prod" })
+    assert_equal \
+      "-t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --target prod",
+      builder.target.build_options.join(" ")
+  end
+
   test "build context" do
     builder = new_builder_command(builder: { "context" => ".." })
     assert_equal \
@@ -112,7 +119,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
   end
 
   test "build with ssh agent socket" do
-    builder = new_builder_command(builder: { "ssh" => 'default=$SSH_AUTH_SOCK' })
+    builder = new_builder_command(builder: { "ssh" => "default=$SSH_AUTH_SOCK" })
 
     assert_equal \
       "-t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --ssh default=$SSH_AUTH_SOCK",
@@ -123,8 +130,40 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     assert_equal "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:123 | grep -x app || (echo \"Image dhh/app:123 is missing the 'service' label\" && exit 1)", new_builder_command.validate_image.join(" ")
   end
 
+  test "multiarch context build" do
+    builder = new_builder_command(builder: { "context" => "./foo" })
+    assert_equal \
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo",
+      builder.push.join(" ")
+  end
+
+  test "native context build" do
+    builder = new_builder_command(builder: { "multiarch" => false, "context" => "./foo" })
+    assert_equal \
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo && docker push dhh/app:123 && docker push dhh/app:latest",
+      builder.push.join(" ")
+  end
+
+  test "cached context build" do
+    builder = new_builder_command(builder: { "multiarch" => false, "context" => "./foo", "cache" => { "type" => "gha" } })
+    assert_equal \
+      "docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile ./foo",
+      builder.push.join(" ")
+  end
+
+  test "remote context build" do
+    builder = new_builder_command(builder: { "remote" => { "arch" => "amd64" }, "context" => "./foo" })
+    assert_equal \
+      "docker buildx build --push --platform linux/amd64 --builder kamal-app-native-remote -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo",
+      builder.push.join(" ")
+  end
+
   private
     def new_builder_command(additional_config = {})
       Kamal::Commands::Builder.new(Kamal::Configuration.new(@config.merge(additional_config), version: "123"))
     end
+
+    def build_directory
+      "#{Dir.tmpdir}/kamal-clones/app/kamal/"
+    end
 end

test/commands/healthcheck_test.rb

@@ -1,114 +0,0 @@
-require "test_helper"
-
-class CommandsHealthcheckTest < ActiveSupport::TestCase
-  setup do
-    @config = {
-      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
-      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
-    }
-  end
-
-  test "run" do
-    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "run with custom port" do
-    @config[:healthcheck] = { "port" => 3001 }
-
-    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3001 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3001/up || exit 1\" --health-interval \"1s\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "run with destination" do
-    @destination = "staging"
-
-    assert_equal \
-      "docker run --detach --name healthcheck-app-staging-123 --publish 3999:3000 --label service=healthcheck-app-staging -e KAMAL_CONTAINER_NAME=\"healthcheck-app-staging\" --env-file .kamal/env/roles/app-web-staging.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "run with custom healthcheck" do
-    @config[:healthcheck] = { "cmd" => "/bin/up" }
-
-    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/up\" --health-interval \"1s\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "run with custom options" do
-    @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere" } } }
-    @config[:healthcheck] = { "exposed_port" => 4999 }
-    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 4999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "status" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'",
-      new_command.status.join(" ")
-  end
-
-  test "container_health_log" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker inspect --format '{{json .State.Health}}'",
-      new_command.container_health_log.join(" ")
-  end
-
-  test "stop" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker stop",
-      new_command.stop.join(" ")
-  end
-
-  test "stop with destination" do
-    @destination = "staging"
-
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-staging-123$ --quiet | xargs docker stop",
-      new_command.stop.join(" ")
-  end
-
-  test "remove" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker container rm",
-      new_command.remove.join(" ")
-  end
-
-  test "remove with destination" do
-    @destination = "staging"
-
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-staging-123$ --quiet | xargs docker container rm",
-      new_command.remove.join(" ")
-  end
-
-  test "logs" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker logs --tail 50 2>&1",
-      new_command.logs.join(" ")
-  end
-
-  test "logs with custom lines number" do
-    @config[:healthcheck] = { "log_lines" => 150 }
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker logs --tail 150 2>&1",
-      new_command.logs.join(" ")
-  end
-
-  test "logs with destination" do
-    @destination = "staging"
-
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-staging-123$ --quiet | xargs docker logs --tail 50 2>&1",
-      new_command.logs.join(" ")
-  end
-
-  private
-    def new_command
-      Kamal::Commands::Healthcheck.new(Kamal::Configuration.new(@config, destination: @destination, version: "123"))
-    end
-end

test/commands/lock_test.rb

@@ -10,24 +10,24 @@ class CommandsLockTest < ActiveSupport::TestCase
 
   test "status" do
     assert_equal \
-      "stat .kamal/lock-app > /dev/null && cat .kamal/lock-app/details | base64 -d",
+      "stat .kamal/locks/app-production > /dev/null && cat .kamal/locks/app-production/details | base64 -d",
       new_command.status.join(" ")
   end
 
   test "acquire" do
     assert_match \
-      %r{mkdir \.kamal/lock-app && echo ".*" > \.kamal/lock-app/details}m,
+      %r{mkdir \.kamal/locks/app-production && echo ".*" > \.kamal/locks/app-production/details}m,
       new_command.acquire("Hello", "123").join(" ")
   end
 
   test "release" do
     assert_match \
-      "rm .kamal/lock-app/details && rm -r .kamal/lock-app",
+      "rm .kamal/locks/app-production/details && rm -r .kamal/locks/app-production",
       new_command.release.join(" ")
   end
 
   private
     def new_command
-      Kamal::Commands::Lock.new(Kamal::Configuration.new(@config, version: "123"))
+      Kamal::Commands::Lock.new(Kamal::Configuration.new(@config, version: "123", destination: "production"))
     end
 end

test/commands/traefik_test.rb

@@ -37,7 +37,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
       "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
-    @config[:traefik]["options"] = {"publish" => %w[9000:9000 9001:9001]}
+    @config[:traefik]["options"] = { "publish" => %w[9000:9000 9001:9001] }
     assert_equal \
       "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
@@ -48,7 +48,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
       "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
-    @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json] }
+    @config[:traefik]["options"] = { "volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json] }
     assert_equal \
       "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
@@ -59,7 +59,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
       "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
-    @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json], "publish" => %w[8080:8080], "memory" => "512m"}
+    @config[:traefik]["options"] = { "volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json], "publish" => %w[8080:8080], "memory" => "512m" }
     assert_equal \
       "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
@@ -138,7 +138,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
   test "traefik logs since 2h" do
     assert_equal \
       "docker logs traefik  --since 2h --timestamps 2>&1",
-      new_command.logs(since: '2h').join(" ")
+      new_command.logs(since: "2h").join(" ")
   end
 
   test "traefik logs last 10 lines" do
@@ -150,7 +150,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
   test "traefik logs with grep hello!" do
     assert_equal \
       "docker logs traefik --timestamps 2>&1 | grep 'hello!'",
-      new_command.logs(grep: 'hello!').join(" ")
+      new_command.logs(grep: "hello!").join(" ")
   end
 
   test "traefik remove container" do
@@ -174,17 +174,13 @@ class CommandsTraefikTest < ActiveSupport::TestCase
   test "traefik follow logs with grep hello!" do
     assert_equal \
       "ssh -t root@1.1.1.1 -p 22 'docker logs traefik --timestamps --tail 10 --follow 2>&1 | grep \"hello!\"'",
-      new_command.follow_logs(host: @config[:servers].first, grep: 'hello!')
+      new_command.follow_logs(host: @config[:servers].first, grep: "hello!")
   end
 
-  test "env_file" do
+  test "secrets io" do
     @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
 
-    assert_equal "EXAMPLE_API_KEY=456\n", new_command.env_file.to_s
+    assert_equal "EXAMPLE_API_KEY=456\n", new_command.env.secrets_io.string
-  end
-
-  test "host_env_file_path" do
-    assert_equal ".kamal/env/traefik/traefik.env", new_command.host_env_file_path
   end
 
   test "make_env_directory" do

test/configuration/accessory_test.rb

@@ -20,7 +20,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
             },
             "secret" => [
               "MYSQL_ROOT_PASSWORD"
-            ],
+            ]
           },
           "files" => [
             "config/mysql/my.cnf:/etc/mysql/my.cnf",
@@ -82,9 +82,9 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
   end
 
   test "host" do
-    assert_equal ["1.1.1.5"], @config.accessory(:mysql).hosts
+    assert_equal [ "1.1.1.5" ], @config.accessory(:mysql).hosts
-    assert_equal ["1.1.1.6", "1.1.1.7"], @config.accessory(:redis).hosts
+    assert_equal [ "1.1.1.6", "1.1.1.7" ], @config.accessory(:redis).hosts
-    assert_equal ["1.1.1.1", "1.1.1.2"], @config.accessory(:monitoring).hosts
+    assert_equal [ "1.1.1.1", "1.1.1.2" ], @config.accessory(:monitoring).hosts
   end
 
   test "missing host" do
@@ -107,40 +107,40 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
     assert_equal "Specify one of `host`, `hosts` or `roles` for accessory `mysql`", exception.message
   end
 
+  test "all hosts" do
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4", "1.1.1.5", "1.1.1.6", "1.1.1.7" ], @config.all_hosts
+  end
+
   test "label args" do
-    assert_equal ["--label", "service=\"app-mysql\""], @config.accessory(:mysql).label_args
+    assert_equal [ "--label", "service=\"app-mysql\"" ], @config.accessory(:mysql).label_args
-    assert_equal ["--label", "service=\"app-redis\"", "--label", "cache=\"true\""], @config.accessory(:redis).label_args
+    assert_equal [ "--label", "service=\"app-redis\"", "--label", "cache=\"true\"" ], @config.accessory(:redis).label_args
   end
 
   test "env args" do
-    assert_equal ["--env-file", ".kamal/env/accessories/app-mysql.env"], @config.accessory(:mysql).env_args
+    assert_equal [ "--env-file", ".kamal/env/accessories/app-mysql.env", "--env", "MYSQL_ROOT_HOST=\"%\"" ], @config.accessory(:mysql).env_args
-    assert_equal ["--env-file", ".kamal/env/accessories/app-redis.env"], @config.accessory(:redis).env_args
+    assert_equal [ "--env-file", ".kamal/env/accessories/app-redis.env", "--env", "SOMETHING=\"else\"" ], @config.accessory(:redis).env_args
   end
 
-  test "env file with secret" do
+  test "env with secrets" do
     ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
 
-    expected = <<~ENV
+    expected_secrets_file = <<~ENV
       MYSQL_ROOT_PASSWORD=secret123
-      MYSQL_ROOT_HOST=%
     ENV
 
-    assert_equal expected, @config.accessory(:mysql).env_file.to_s
+    assert_equal expected_secrets_file, @config.accessory(:mysql).env.secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/accessories/app-mysql.env", "--env", "MYSQL_ROOT_HOST=\"%\"" ], @config.accessory(:mysql).env_args
   ensure
     ENV["MYSQL_ROOT_PASSWORD"] = nil
   end
 
-  test "host_env_directory" do
+  test "env secrets path" do
-    assert_equal ".kamal/env/accessories", @config.accessory(:mysql).host_env_directory
+    assert_equal ".kamal/env/accessories/app-mysql.env", @config.accessory(:mysql).env.secrets_file
-  end
-
-  test "host_env_file_path" do
-    assert_equal ".kamal/env/accessories/app-mysql.env", @config.accessory(:mysql).host_env_file_path
   end
 
   test "volume args" do
-    assert_equal ["--volume", "$PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf", "--volume", "$PWD/app-mysql/docker-entrypoint-initdb.d/structure.sql:/docker-entrypoint-initdb.d/structure.sql", "--volume", "$PWD/app-mysql/data:/var/lib/mysql"], @config.accessory(:mysql).volume_args
+    assert_equal [ "--volume", "$PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf", "--volume", "$PWD/app-mysql/docker-entrypoint-initdb.d/structure.sql:/docker-entrypoint-initdb.d/structure.sql", "--volume", "$PWD/app-mysql/data:/var/lib/mysql" ], @config.accessory(:mysql).volume_args
-    assert_equal ["--volume", "/var/lib/redis:/data"], @config.accessory(:redis).volume_args
+    assert_equal [ "--volume", "/var/lib/redis:/data" ], @config.accessory(:redis).volume_args
   end
 
   test "dynamic file expansion" do
@@ -153,15 +153,15 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
 
   test "directory with a relative path" do
     @deploy[:accessories]["mysql"]["directories"] = [ "data:/var/lib/mysql" ]
-    assert_equal({"$PWD/app-mysql/data"=>"/var/lib/mysql"}, @config.accessory(:mysql).directories)
+    assert_equal({ "$PWD/app-mysql/data"=>"/var/lib/mysql" }, @config.accessory(:mysql).directories)
   end
 
   test "directory with an absolute path" do
     @deploy[:accessories]["mysql"]["directories"] = [ "/var/data/mysql:/var/lib/mysql" ]
-    assert_equal({"/var/data/mysql"=>"/var/lib/mysql"}, @config.accessory(:mysql).directories)
+    assert_equal({ "/var/data/mysql"=>"/var/lib/mysql" }, @config.accessory(:mysql).directories)
   end
 
   test "options" do
-    assert_equal ["--cpus", "\"4\"", "--memory", "\"2GB\""], @config.accessory(:redis).option_args
+    assert_equal [ "--cpus", "\"4\"", "--memory", "\"2GB\"" ], @config.accessory(:redis).option_args
   end
 end

test/configuration/builder_test.rb

@@ -124,9 +124,9 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase
   end
 
   test "setting secrets" do
-    @deploy_with_builder_option[:builder] = { "secrets" => ["GITHUB_TOKEN"] }
+    @deploy_with_builder_option[:builder] = { "secrets" => [ "GITHUB_TOKEN" ] }
 
-    assert_equal ["GITHUB_TOKEN"], @config_with_builder_option.builder.secrets
+    assert_equal [ "GITHUB_TOKEN" ], @config_with_builder_option.builder.secrets
   end
 
   test "dockerfile" do
@@ -154,8 +154,8 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase
   end
 
   test "setting ssh params" do
-    @deploy_with_builder_option[:builder] = { "ssh" => 'default=$SSH_AUTH_SOCK' }
+    @deploy_with_builder_option[:builder] = { "ssh" => "default=$SSH_AUTH_SOCK" }
 
-    assert_equal 'default=$SSH_AUTH_SOCK', @config_with_builder_option.builder.ssh
+    assert_equal "default=$SSH_AUTH_SOCK", @config_with_builder_option.builder.ssh
   end
 end

test/configuration/env/tags_test.rb

@@ -0,0 +1,112 @@
+require "test_helper"
+
+class ConfigurationEnvTagsTest < ActiveSupport::TestCase
+  setup do
+    @deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+      servers: [ { "1.1.1.1" => "odd" }, { "1.1.1.2" => "even" }, { "1.1.1.3" => [ "odd", "three" ] } ],
+      env: {
+        "clear" => { "REDIS_URL" => "redis://x/y", "THREE" => "false" },
+        "tags" => {
+          "odd" => { "TYPE" => "odd" },
+          "even" => { "TYPE" => "even" },
+          "three" => { "THREE" => "true" }
+        }
+      }
+    }
+
+    @config = Kamal::Configuration.new(@deploy)
+
+    @deploy_with_roles = @deploy.dup.merge({
+      servers: {
+        "web" => [ { "1.1.1.1" => "odd" }, "1.1.1.2" ],
+        "workers" => {
+          "hosts" => [ { "1.1.1.3" => [ "odd", "oddjob" ] }, "1.1.1.4" ],
+          "cmd" => "bin/jobs",
+          "env" => {
+            "REDIS_URL" => "redis://a/b",
+            "WEB_CONCURRENCY" => 4
+          }
+        }
+      },
+      env: {
+        "tags" => {
+          "odd" => { "TYPE" => "odd" },
+          "oddjob" => { "TYPE" => "oddjob" }
+        }
+      }
+    })
+
+    @config_with_roles = Kamal::Configuration.new(@deploy_with_roles)
+  end
+
+  test "tags" do
+    assert_equal 3, @config.env_tags.size
+    assert_equal %w[ odd even three ], @config.env_tags.map(&:name)
+    assert_equal({ "TYPE" => "odd" }, @config.env_tag("odd").env.clear)
+    assert_equal({ "TYPE" => "even" }, @config.env_tag("even").env.clear)
+    assert_equal({ "THREE" => "true" }, @config.env_tag("three").env.clear)
+  end
+
+  test "tags with roles" do
+    assert_equal 2, @config_with_roles.env_tags.size
+    assert_equal %w[ odd oddjob ], @config_with_roles.env_tags.map(&:name)
+    assert_equal({ "TYPE" => "odd" }, @config_with_roles.env_tag("odd").env.clear)
+    assert_equal({ "TYPE" => "oddjob" }, @config_with_roles.env_tag("oddjob").env.clear)
+  end
+
+  test "tag overrides env" do
+    assert_equal "false", @config.role("web").env("1.1.1.1").clear["THREE"]
+    assert_equal "true", @config.role("web").env("1.1.1.3").clear["THREE"]
+  end
+
+  test "later tag wins" do
+    deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+      servers: [ { "1.1.1.1" => [ "first", "second" ] } ],
+      env: {
+        "tags" => {
+          "first" => { "TYPE" => "first" },
+          "second" => { "TYPE" => "second" }
+        }
+      }
+    }
+
+    config = Kamal::Configuration.new(deploy)
+    assert_equal "second", config.role("web").env("1.1.1.1").clear["TYPE"]
+  end
+
+  test "tag secret env" do
+    ENV["PASSWORD"] = "hello"
+
+    deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+      servers: [ { "1.1.1.1" => "secrets" } ],
+      env: {
+        "tags" => {
+          "secrets" => { "secret" => [ "PASSWORD" ] }
+        }
+      }
+    }
+
+    config = Kamal::Configuration.new(deploy)
+    assert_equal "hello", config.role("web").env("1.1.1.1").secrets["PASSWORD"]
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "tag clear env" do
+    deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+      servers: [ { "1.1.1.1" => "clearly" } ],
+      env: {
+        "tags" => {
+          "clearly" => { "clear" => { "FOO" => "bar" } }
+        }
+      }
+    }
+
+    config = Kamal::Configuration.new(deploy)
+    assert_equal "bar", config.role("web").env("1.1.1.1").clear["FOO"]
+  end
+end

test/configuration/env_test.rb

@@ -0,0 +1,74 @@
+require "test_helper"
+
+class ConfigurationEnvTest < ActiveSupport::TestCase
+  require "test_helper"
+
+  test "simple" do
+    assert_config \
+      config: { "foo" => "bar", "baz" => "haz" },
+      clear: { "foo" => "bar", "baz" => "haz" },
+      secrets: {}
+  end
+
+  test "clear" do
+    assert_config \
+      config: { "clear" => { "foo" => "bar", "baz" => "haz" } },
+      clear: { "foo" => "bar", "baz" => "haz" },
+      secrets: {}
+  end
+
+  test "secret" do
+    ENV["PASSWORD"] = "hello"
+    env = Kamal::Configuration::Env.from_config config: { "secret" => [ "PASSWORD" ] }
+
+    assert_config \
+      config: { "secret" => [ "PASSWORD" ] },
+      clear: {},
+      secrets: { "PASSWORD" => "hello" }
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "missing secret" do
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_raises(KeyError) { Kamal::Configuration::Env.from_config(config: { "secret" => [ "PASSWORD" ] }).secrets }
+  end
+
+  test "secret and clear" do
+    ENV["PASSWORD"] = "hello"
+    config = {
+      "secret" => [ "PASSWORD" ],
+      "clear" => {
+        "foo" => "bar",
+        "baz" => "haz"
+      }
+    }
+
+    assert_config \
+      config: config,
+      clear: { "foo" => "bar", "baz" => "haz" },
+      secrets: { "PASSWORD" => "hello" }
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "stringIO conversion" do
+    env = {
+      "foo" => "bar",
+      "baz" => "haz"
+    }
+
+    assert_equal "foo=bar\nbaz=haz\n", \
+      StringIO.new(Kamal::EnvFile.new(env)).read
+  end
+
+  private
+    def assert_config(config:, clear:, secrets:)
+      env = Kamal::Configuration::Env.from_config config: config
+      assert_equal clear, env.clear
+      assert_equal secrets, env.secrets
+    end
+end

test/configuration/role_test.rb

@@ -38,11 +38,11 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
   end
 
   test "label args" do
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"workers\"" ], @config_with_roles.role(:workers).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"workers\"", "--label", "destination" ], @config_with_roles.role(:workers).label_args
   end
 
   test "special label args for web" do
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.routers.app-web.priority=\"2\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "destination", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.routers.app-web.priority=\"2\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args
   end
 
   test "custom labels" do
@@ -66,18 +66,14 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       c[:servers]["beta"] = { "traefik" => "true", "hosts" => [ "1.1.1.5" ] }
     })
 
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "traefik.http.services.app-beta.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.routers.app-beta.priority=\"2\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "destination", "--label", "traefik.http.services.app-beta.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.routers.app-beta.priority=\"2\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args
   end
 
   test "env overwritten by role" do
-    assert_equal "redis://a/b", @config_with_roles.role(:workers).env["REDIS_URL"]
+    assert_equal "redis://a/b", @config_with_roles.role(:workers).env("1.1.1.3").clear["REDIS_URL"]
 
-    expected_env = <<~ENV
+    assert_equal "\n", @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
-      REDIS_URL=redis://a/b
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
-      WEB_CONCURRENCY=4
-    ENV
-
-    assert_equal expected_env, @config_with_roles.role(:workers).env_file.to_s
   end
 
   test "container name" do
@@ -90,7 +86,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
   end
 
   test "env args" do
-    assert_equal ["--env-file", ".kamal/env/roles/app-workers.env"], @config_with_roles.role(:workers).env_args
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
   end
 
   test "env secret overwritten by role" do
@@ -116,14 +112,13 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     ENV["REDIS_PASSWORD"] = "secret456"
     ENV["DB_PASSWORD"] = "secret&\"123"
 
-    expected = <<~ENV
+    expected_secrets_file = <<~ENV
       REDIS_PASSWORD=secret456
       DB_PASSWORD=secret&\"123
-      REDIS_URL=redis://a/b
-      WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected, @config_with_roles.role(:workers).env_file.to_s
+    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
   ensure
     ENV["REDIS_PASSWORD"] = nil
     ENV["DB_PASSWORD"] = nil
@@ -142,13 +137,12 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
 
     ENV["DB_PASSWORD"] = "secret123"
 
-    expected = <<~ENV
+    expected_secrets_file = <<~ENV
       DB_PASSWORD=secret123
-      REDIS_URL=redis://a/b
-      WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected, @config_with_roles.role(:workers).env_file.to_s
+    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
   ensure
     ENV["DB_PASSWORD"] = nil
   end
@@ -165,13 +159,12 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
 
     ENV["REDIS_PASSWORD"] = "secret456"
 
-    expected = <<~ENV
+    expected_secrets_file = <<~ENV
       REDIS_PASSWORD=secret456
-      REDIS_URL=redis://a/b
-      WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected, @config_with_roles.role(:workers).env_file.to_s
+    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
   ensure
     ENV["REDIS_PASSWORD"] = nil
   end
@@ -188,33 +181,29 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
 
     @deploy_with_roles[:servers]["workers"]["env"] = {
       "clear" => {
-        "REDIS_URL" => "redis://c/d",
+        "REDIS_URL" => "redis://c/d"
-      },
+      }
     }
 
     ENV["REDIS_PASSWORD"] = "secret456"
 
-    expected = <<~ENV
+    expected_secrets_file = <<~ENV
       REDIS_PASSWORD=secret456
-      REDIS_URL=redis://c/d
     ENV
 
-    assert_equal expected, @config_with_roles.role(:workers).env_file.to_s
+    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://c/d\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
   ensure
     ENV["REDIS_PASSWORD"] = nil
   end
 
-  test "host_env_directory" do
+  test "env secrets_file" do
-    assert_equal ".kamal/env/roles", @config_with_roles.role(:workers).host_env_directory
+    assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).env("1.1.1.3").secrets_file
-  end
-
-  test "host_env_file_path" do
-    assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).host_env_file_path
   end
 
   test "uses cord" do
     assert @config_with_roles.role(:web).uses_cord?
-    assert !@config_with_roles.role(:workers).uses_cord?
+    assert_not @config_with_roles.role(:workers).uses_cord?
   end
 
   test "cord host file" do
@@ -238,28 +227,28 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     assert_nil @config_with_roles.role(:workers).asset_volume_args
     assert_nil @config_with_roles.role(:web).asset_path
     assert_nil @config_with_roles.role(:workers).asset_path
-    assert !@config_with_roles.role(:web).assets?
+    assert_not @config_with_roles.role(:web).assets?
-    assert !@config_with_roles.role(:workers).assets?
+    assert_not @config_with_roles.role(:workers).assets?
 
     config_with_assets = Kamal::Configuration.new(@deploy_with_roles.dup.tap { |c|
       c[:asset_path] = "foo"
     })
     assert_equal "foo", config_with_assets.role(:web).asset_path
     assert_equal "foo", config_with_assets.role(:workers).asset_path
-    assert_equal ["--volume", "$(pwd)/.kamal/assets/volumes/app-web-12345:foo"], config_with_assets.role(:web).asset_volume_args
+    assert_equal [ "--volume", "$(pwd)/.kamal/assets/volumes/app-web-12345:foo" ], config_with_assets.role(:web).asset_volume_args
     assert_nil config_with_assets.role(:workers).asset_volume_args
     assert config_with_assets.role(:web).assets?
-    assert !config_with_assets.role(:workers).assets?
+    assert_not config_with_assets.role(:workers).assets?
 
     config_with_assets = Kamal::Configuration.new(@deploy_with_roles.dup.tap { |c|
       c[:servers]["web"] = { "hosts" => [ "1.1.1.1", "1.1.1.2" ], "asset_path" => "bar" }
     })
     assert_equal "bar", config_with_assets.role(:web).asset_path
     assert_nil config_with_assets.role(:workers).asset_path
-    assert_equal ["--volume", "$(pwd)/.kamal/assets/volumes/app-web-12345:bar"], config_with_assets.role(:web).asset_volume_args
+    assert_equal [ "--volume", "$(pwd)/.kamal/assets/volumes/app-web-12345:bar" ], config_with_assets.role(:web).asset_volume_args
     assert_nil config_with_assets.role(:workers).asset_volume_args
     assert config_with_assets.role(:web).assets?
-    assert !config_with_assets.role(:workers).assets?
+    assert_not config_with_assets.role(:workers).assets?
 
   ensure
     ENV.delete("VERSION")

test/configuration/ssh_test.rb

@@ -7,7 +7,7 @@ class ConfigurationSshTest < ActiveSupport::TestCase
       registry: { "username" => "dhh", "password" => "secret" },
       env: { "REDIS_URL" => "redis://x/y" },
       servers: [ "1.1.1.1", "1.1.1.2" ],
-      volumes: ["/local/path:/container/path"]
+      volumes: [ "/local/path:/container/path" ]
     }
 
     @config = Kamal::Configuration.new(@deploy)

test/configuration/sshkit_test.rb

@@ -7,7 +7,7 @@ class ConfigurationSshkitTest < ActiveSupport::TestCase
       registry: { "username" => "dhh", "password" => "secret" },
       env: { "REDIS_URL" => "redis://x/y" },
       servers: [ "1.1.1.1", "1.1.1.2" ],
-      volumes: ["/local/path:/container/path"]
+      volumes: [ "/local/path:/container/path" ]
     }
 
     @config = Kamal::Configuration.new(@deploy)

test/configuration/volume_test.rb

@@ -3,11 +3,11 @@ require "test_helper"
 class ConfigurationVolumeTest < ActiveSupport::TestCase
   test "docker args absolute" do
     volume = Kamal::Configuration::Volume.new(host_path: "/root/foo/bar", container_path: "/assets")
-    assert_equal ["--volume", "/root/foo/bar:/assets"], volume.docker_args
+    assert_equal [ "--volume", "/root/foo/bar:/assets" ], volume.docker_args
   end
 
   test "docker args relative" do
     volume = Kamal::Configuration::Volume.new(host_path: "foo/bar", container_path: "/assets")
-    assert_equal ["--volume", "$(pwd)/foo/bar:/assets"], volume.docker_args
+    assert_equal [ "--volume", "$(pwd)/foo/bar:/assets" ], volume.docker_args
   end
 end

test/configuration_test.rb

@@ -10,7 +10,7 @@ class ConfigurationTest < ActiveSupport::TestCase
       registry: { "username" => "dhh", "password" => "secret" },
       env: { "REDIS_URL" => "redis://x/y" },
       servers: [ "1.1.1.1", "1.1.1.2" ],
-      volumes: ["/local/path:/container/path"]
+      volumes: [ "/local/path:/container/path" ]
     }
 
     @config = Kamal::Configuration.new(@deploy)
@@ -44,6 +44,7 @@ class ConfigurationTest < ActiveSupport::TestCase
 
   test "service name valid" do
     assert Kamal::Configuration.new(@deploy.tap { _1[:service] = "hey-app1_primary" }).valid?
+    assert Kamal::Configuration.new(@deploy.tap { _1[:service] = "MyApp" }).valid?
   end
 
   test "service name invalid" do
@@ -64,7 +65,7 @@ class ConfigurationTest < ActiveSupport::TestCase
   end
 
   test "all hosts" do
-    assert_equal [ "1.1.1.1", "1.1.1.2"], @config.all_hosts
+    assert_equal [ "1.1.1.1", "1.1.1.2" ], @config.all_hosts
     assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], @config_with_roles.all_hosts
   end
 
@@ -82,11 +83,20 @@ class ConfigurationTest < ActiveSupport::TestCase
     assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], config.traefik_hosts
   end
 
+  test "filtered traefik hosts" do
+    assert_equal [ "1.1.1.1", "1.1.1.2" ], @config_with_roles.traefik_hosts
+
+    @deploy_with_roles[:servers]["workers"]["traefik"] = true
+    config = Kamal::Configuration.new(@deploy_with_roles)
+
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], config.traefik_hosts
+  end
+
   test "version no git repo" do
     ENV.delete("VERSION")
 
     Kamal::Git.expects(:used?).returns(nil)
-    error = assert_raises(RuntimeError) { @config.version}
+    error = assert_raises(RuntimeError) { @config.version }
     assert_match /no git repository found/, error.message
   end
 
@@ -103,7 +113,17 @@ class ConfigurationTest < ActiveSupport::TestCase
 
     Kamal::Git.expects(:revision).returns("git-version")
     Kamal::Git.expects(:uncommitted_changes).returns("M   file\n")
-    assert_match /^git-version_uncommitted_[0-9a-f]{16}$/, @config.version
+    assert_equal "git-version", @config.version
+  end
+
+  test "version from uncommitted context" do
+    ENV.delete("VERSION")
+
+    config = Kamal::Configuration.new(@deploy.tap { |c| c[:builder] = { "context" => "." } })
+
+    Kamal::Git.expects(:revision).returns("git-version")
+    Kamal::Git.expects(:uncommitted_changes).returns("M   file\n")
+    assert_match /^git-version_uncommitted_[0-9a-f]{16}$/, config.version
   end
 
   test "version from env" do
@@ -186,21 +206,21 @@ class ConfigurationTest < ActiveSupport::TestCase
   end
 
   test "volume_args" do
-    assert_equal ["--volume", "/local/path:/container/path"], @config.volume_args
+    assert_equal [ "--volume", "/local/path:/container/path" ], @config.volume_args
   end
 
   test "logging args default" do
-    assert_equal ["--log-opt", "max-size=\"10m\""], @config.logging_args
+    assert_equal [ "--log-opt", "max-size=\"10m\"" ], @config.logging_args
   end
 
   test "logging args with configured options" do
     config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!(logging: { "options" => { "max-size" => "100m", "max-file" => 5 } }) })
-    assert_equal ["--log-opt", "max-size=\"100m\"", "--log-opt", "max-file=\"5\""], @config.logging_args
+    assert_equal [ "--log-opt", "max-size=\"100m\"", "--log-opt", "max-file=\"5\"" ], @config.logging_args
   end
 
   test "logging args with configured driver and options" do
     config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!(logging: { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => 5 } }) })
-    assert_equal ["--log-driver", "\"local\"", "--log-opt", "max-size=\"100m\"", "--log-opt", "max-file=\"5\""], @config.logging_args
+    assert_equal [ "--log-driver", "\"local\"", "--log-opt", "max-size=\"100m\"", "--log-opt", "max-file=\"5\"" ], @config.logging_args
   end
 
   test "erb evaluation of yml config" do
@@ -240,19 +260,19 @@ class ConfigurationTest < ActiveSupport::TestCase
 
   test "to_h" do
     expected_config = \
-      { :roles=>["web"],
+      { roles: [ "web" ],
-        :hosts=>["1.1.1.1", "1.1.1.2"],
+        hosts: [ "1.1.1.1", "1.1.1.2" ],
-        :primary_host=>"1.1.1.1",
+        primary_host: "1.1.1.1",
-        :version=>"missing",
+        version: "missing",
-        :repository=>"dhh/app",
+        repository: "dhh/app",
-        :absolute_image=>"dhh/app:missing",
+        absolute_image: "dhh/app:missing",
-        :service_with_version=>"app-missing",
+        service_with_version: "app-missing",
-        :ssh_options=>{ :user=>"root", port: 22, log_level: :fatal, keepalive: true, keepalive_interval: 30 },
+        ssh_options: { user: "root", port: 22, log_level: :fatal, keepalive: true, keepalive_interval: 30 },
-        :sshkit=>{},
+        sshkit: {},
-        :volume_args=>["--volume", "/local/path:/container/path"],
+        volume_args: [ "--volume", "/local/path:/container/path" ],
-        :builder=>{},
+        builder: {},
-        :logging=>["--log-opt", "max-size=\"10m\""],
+        logging: [ "--log-opt", "max-size=\"10m\"" ],
-        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }}
+        healthcheck: { "path"=>"/up", "port"=>3000, "max_attempts" => 7, "cord" => "/tmp/kamal-cord", "log_lines" => 50 } }
 
     assert_equal expected_config, @config.to_h
   end
@@ -304,7 +324,7 @@ class ConfigurationTest < ActiveSupport::TestCase
 
     config = Kamal::Configuration.new(@deploy_with_roles.deep_merge({
       servers: { "alternate_web" => { "hosts" => [ "1.1.1.4", "1.1.1.5" ] } },
-      primary_role: "alternate_web" } ))
+      primary_role: "alternate_web" }))
 
 
     assert_equal "alternate_web", config.primary_role.name

test/env_file_test.rb

@@ -1,7 +1,7 @@
 require "test_helper"
 
 class EnvFileTest < ActiveSupport::TestCase
-  test "env file simple" do
+  test "to_s" do
     env = {
       "foo" => "bar",
       "baz" => "haz"
@@ -11,80 +11,54 @@ class EnvFileTest < ActiveSupport::TestCase
       Kamal::EnvFile.new(env).to_s
   end
 
-  test "env file clear" do
+  test "to_str won't escape chinese characters" do
     env = {
-      "clear" => {
+      "foo" => '你好 means hello, "欢迎" means welcome, that\'s simple! 😃 {smile}'
-        "foo" => "bar",
-        "baz" => "haz"
-      }
     }
 
-    assert_equal "foo=bar\nbaz=haz\n", \
+    assert_equal "foo=你好 means hello, \"欢迎\" means welcome, that's simple! 😃 {smile}\n",
       Kamal::EnvFile.new(env).to_s
   end
 
-  test "env file empty" do
+  test "to_s won't escape japanese characters" do
+    env = {
+      "foo" => 'こんにちは means hello, "ようこそ" means welcome, that\'s simple! 😃 {smile}'
+    }
+
+    assert_equal "foo=こんにちは means hello, \"ようこそ\" means welcome, that's simple! 😃 {smile}\n", \
+      Kamal::EnvFile.new(env).to_s
+  end
+
+  test "to_s won't escape korean characters" do
+    env = {
+      "foo" => '안녕하세요 means hello, "어서 오십시오" means welcome, that\'s simple! 😃 {smile}'
+    }
+
+    assert_equal "foo=안녕하세요 means hello, \"어서 오십시오\" means welcome, that's simple! 😃 {smile}\n", \
+      Kamal::EnvFile.new(env).to_s
+  end
+
+  test "to_s empty" do
     assert_equal "\n", Kamal::EnvFile.new({}).to_s
   end
 
-  test "env file secret" do
+  test "to_s escaped newline" do
-    ENV["PASSWORD"] = "hello"
     env = {
-      "secret" => [ "PASSWORD" ]
+      "foo" => "hello\\nthere"
     }
 
-    assert_equal "PASSWORD=hello\n", \
+    assert_equal "foo=hello\\\\nthere\n", \
       Kamal::EnvFile.new(env).to_s
   ensure
     ENV.delete "PASSWORD"
   end
 
-  test "env file secret escaped newline" do
+  test "to_s newline" do
-    ENV["PASSWORD"] = "hello\\nthere"
     env = {
-      "secret" => [ "PASSWORD" ]
+      "foo" => "hello\nthere"
     }
 
-    assert_equal "PASSWORD=hello\\\\nthere\n", \
+    assert_equal "foo=hello\\nthere\n", \
-      Kamal::EnvFile.new(env).to_s
-  ensure
-    ENV.delete "PASSWORD"
-  end
-
-  test "env file secret newline" do
-    ENV["PASSWORD"] = "hello\nthere"
-    env = {
-      "secret" => [ "PASSWORD" ]
-    }
-
-    assert_equal "PASSWORD=hello\\nthere\n", \
-      Kamal::EnvFile.new(env).to_s
-  ensure
-    ENV.delete "PASSWORD"
-  end
-
-  test "env file missing secret" do
-    env = {
-      "secret" => [ "PASSWORD" ]
-    }
-
-    assert_raises(KeyError) { Kamal::EnvFile.new(env).to_s }
-
-  ensure
-    ENV.delete "PASSWORD"
-  end
-
-  test "env file secret and clear" do
-    ENV["PASSWORD"] = "hello"
-    env = {
-      "secret" => [ "PASSWORD" ],
-      "clear" => {
-        "foo" => "bar",
-        "baz" => "haz"
-      }
-    }
-
-    assert_equal "PASSWORD=hello\nfoo=bar\nbaz=haz\n", \
       Kamal::EnvFile.new(env).to_s
   ensure
     ENV.delete "PASSWORD"

test/fixtures/deploy_with_env_tags.yml

@@ -0,0 +1,28 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - 1.1.1.1: site1
+    - 1.1.1.2: [ site1 experimental ]
+    - 1.2.1.1: site2
+    - 1.2.1.2: site2
+  workers:
+    - 1.1.1.3: site1
+    - 1.1.1.4: site1
+    - 1.2.1.3: site2
+    - 1.2.1.4: [ site2 experimental ]
+env:
+  clear:
+    TEST: "root"
+    EXPERIMENT: "disabled"
+  tags:
+    site1:
+      SITE: site1
+    site2:
+      SITE: site2
+    experimental:
+      EXPERIMENT: "enabled"
+
+registry:
+  username: user
+  password: pw

test/fixtures/deploy_with_remote_builder_and_custom_ports.yml

@@ -0,0 +1,45 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+registry:
+  username: user
+  password: pw
+
+accessories:
+  mysql:
+    image: mysql:5.7
+    host: 1.1.1.3
+    port: 3306
+    env:
+      clear:
+        MYSQL_ROOT_HOST: '%'
+      secret:
+        - MYSQL_ROOT_PASSWORD
+    files:
+      - test/fixtures/files/my.cnf:/etc/mysql/my.cnf
+    directories:
+      - data:/var/lib/mysql
+  redis:
+    image: redis:latest
+    roles:
+      - web
+    port: 6379
+    directories:
+      - data:/data
+
+readiness_delay: 0
+
+ssh:
+  user: root
+  port: 22
+
+builder:
+  remote:
+    arch: amd64
+    host: ssh://app@1.1.1.5:2122

test/fixtures/deploy_with_two_roles_one_host.yml

@@ -0,0 +1,15 @@
+service: app
+image: dhh/app
+servers:
+  workers:
+    hosts:
+      - 1.1.1.1
+  web:
+    hosts:
+      - 1.1.1.1
+env:
+  REDIS_URL: redis://x/y
+registry:
+  server: registry.digitalocean.com
+  username: user
+  password: pw

test/fixtures/deploy_with_uncommon_hostnames.yml

@@ -0,0 +1,8 @@
+service: app
+image: dhh/app
+servers:
+  - "this-hostname-with-random-part-is-too-long.example.com"
+  - "this-hostname-is-really-unacceptably-long-to-be-honest.example.com"
+registry:
+  username: user
+  password: pw

test/fixtures/deploy_without_clone.yml

@@ -0,0 +1,39 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+registry:
+  username: user
+  password: pw
+
+accessories:
+  mysql:
+    image: mysql:5.7
+    host: 1.1.1.3
+    port: 3306
+    env:
+      clear:
+        MYSQL_ROOT_HOST: '%'
+      secret:
+        - MYSQL_ROOT_PASSWORD
+    files:
+      - test/fixtures/files/my.cnf:/etc/mysql/my.cnf
+    directories:
+      - data:/var/lib/mysql
+  redis:
+    image: redis:latest
+    roles:
+      - web
+    port: 6379
+    directories:
+      - data:/data
+
+readiness_delay: 0
+
+builder:
+  context: "."

test/integration/accessory_test.rb

@@ -31,7 +31,7 @@ class AccessoryTest < IntegrationTest
     end
 
     def assert_accessory_not_running(name)
-      refute_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
+      assert_no_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
     end
 
     def accessory_details(name)

test/integration/broken_deploy_test.rb

@@ -0,0 +1,34 @@
+require_relative "integration_test"
+
+class BrokenDeployTest < IntegrationTest
+  test "deploying a bad image" do
+    @app = "app_with_roles"
+
+    kamal :envify
+
+    first_version = latest_app_version
+
+    kamal :deploy
+
+    assert_app_is_up version: first_version
+    assert_container_running host: :vm3, name: "app-workers-#{first_version}"
+
+    second_version = break_app
+
+    output = kamal :deploy, raise_on_error: false, capture: true
+
+    assert_failed_deploy output
+    assert_app_is_up version: first_version
+    assert_container_running host: :vm3, name: "app-workers-#{first_version}"
+    assert_container_not_running host: :vm3, name: "app-workers-#{second_version}"
+  end
+
+  private
+    def assert_failed_deploy(output)
+      assert_match "Waiting for the first healthy web container before booting workers on vm3...", output
+      assert_match /First web container is unhealthy on vm[12], not booting other roles/, output
+      assert_match "First web container is unhealthy, not booting workers on vm3", output
+      assert_match "nginx: [emerg] unexpected end of file, expecting \";\" or \"}\" in /etc/nginx/conf.d/default.conf:2", output
+      assert_match 'ERROR {"Status":"unhealthy","FailingStreak":0,"Log":[]}', output
+    end
+end

test/integration/docker-compose.yml

@@ -18,7 +18,7 @@ services:
     build:
       context: docker/deployer
     environment:
-      - TEST_ID=${TEST_ID}
+      - TEST_ID=${TEST_ID:-}
     volumes:
       - ../..:/kamal
       - shared:/shared
@@ -50,8 +50,19 @@ services:
     volumes:
       - shared:/shared
 
+  vm3:
+    privileged: true
+    build:
+      context: docker/vm
+    volumes:
+      - shared:/shared
+
   load_balancer:
     build:
       context: docker/load_balancer
     ports:
       - "12345:80"
+    depends_on:
+      - vm1
+      - vm2
+      - vm3

test/integration/docker/deployer/Dockerfile

@@ -1,6 +1,6 @@
 FROM ruby:3.2
 
-WORKDIR /app
+WORKDIR /
 
 ENV VERBOSE=true
 
@@ -17,7 +17,8 @@ RUN echo \
 RUN apt-get update --fix-missing && apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
 
 COPY *.sh .
-COPY app/ .
+COPY app/ app/
+COPY app_with_roles/ app_with_roles/
 
 RUN rm -rf /root/.ssh
 RUN ln -s /shared/ssh /root/.ssh
@@ -25,7 +26,8 @@ RUN mkdir -p /etc/docker/certs.d/registry:4443 && ln -s /shared/certs/domain.crt
 
 RUN git config --global user.email "deployer@example.com"
 RUN git config --global user.name "Deployer"
-RUN git init && echo ".env" >> .gitignore && git add . && git commit -am "Initial version"
+RUN cd app && git init && echo ".env" >> .gitignore && git add . && git commit -am "Initial version"
+RUN cd app_with_roles && git init && echo ".env" >> .gitignore && git add . && git commit -am "Initial version"
 
 HEALTHCHECK --interval=1s CMD pgrep sleep
 

test/integration/docker/deployer/app/.env.erb

@@ -1 +1,2 @@
-SECRET_TOKEN=1234
+SECRET_TOKEN='1234 with "中文"'
+SECRET_TAG='TAGME'

test/integration/docker/deployer/app/config/deploy.yml

@@ -2,12 +2,20 @@ service: app
 image: app
 servers:
   - vm1
-  - vm2
+  - vm2: [ tag1, tag2 ]
 env:
   clear:
-    CLEAR_TOKEN: '4321'
+    CLEAR_TOKEN: 4321
+    CLEAR_TAG: ""
+    HOST_TOKEN: "${HOST_TOKEN}"
   secret:
     - SECRET_TOKEN
+  tags:
+    tag1:
+      CLEAR_TAG: tagged
+    tag2:
+      secret:
+        - SECRET_TAG
 asset_path: /usr/share/nginx/html/versions
 
 registry:
@@ -20,6 +28,7 @@ builder:
     COMMIT_SHA: <%= `git rev-parse HEAD` %>
 healthcheck:
   cmd: wget -qO- http://localhost > /dev/null || exit 1
+  max_attempts: 3
 traefik:
   args:
     accesslog: true
@@ -33,3 +42,4 @@ accessories:
     roles:
       - web
 stop_wait_time: 1
+readiness_delay: 0