Bump version for 1.4.0

Pass around Roles instead of Strings

.github/workflows/ci.yml

@@ -1,5 +1,5 @@
 name: CI
-on: 
+on:
   push:
     branches:
       - main
@@ -12,17 +12,29 @@ jobs:
           - "2.7"
           - "3.1"
           - "3.2"
+          - "3.3"
         gemfile:
           - Gemfile
+          - gemfiles/ruby_2.7.gemfile
           - gemfiles/rails_edge.gemfile
-        continue-on-error: [false]
+        exclude:
+          - ruby-version: "2.7"
+            gemfile: Gemfile
+          - ruby-version: "2.7"
+            gemfile: gemfiles/rails_edge.gemfile
+          - ruby-version: "3.1"
+            gemfile: gemfiles/ruby_2.7.gemfile
+          - ruby-version: "3.2"
+            gemfile: gemfiles/ruby_2.7.gemfile
+          - ruby-version: "3.3"
+            gemfile: gemfiles/ruby_2.7.gemfile
     name: ${{ format('Tests (Ruby {0})', matrix.ruby-version) }}
     runs-on: ubuntu-latest
-    continue-on-error: ${{ matrix.continue-on-error }}
+    continue-on-error: true
     env:
       BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install Ruby
         uses: ruby/setup-ruby@v1

Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    kamal (0.16.1)
+    kamal (1.4.0)
       activesupport (>= 7.0)
+      base64 (~> 0.2)
       bcrypt_pbkdf (~> 1.0)
       concurrent-ruby (~> 1.2)
       dotenv (~> 2.8)
@@ -15,82 +16,111 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.0.4.3)
-      actionview (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      rack (~> 2.0, >= 2.2.0)
+    actionpack (7.1.2)
+      actionview (= 7.1.2)
+      activesupport (= 7.1.2)
+      nokogiri (>= 1.8.5)
+      racc
+      rack (>= 2.2.4)
+      rack-session (>= 1.0.1)
       rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (7.0.4.3)
-      activesupport (= 7.0.4.3)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+    actionview (7.1.2)
+      activesupport (= 7.1.2)
       builder (~> 3.1)
-      erubi (~> 1.4)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activesupport (7.0.4.3)
+      erubi (~> 1.11)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+    activesupport (7.1.2)
+      base64
+      bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
+      mutex_m
       tzinfo (~> 2.0)
+    base64 (0.2.0)
     bcrypt_pbkdf (1.1.0)
+    bigdecimal (3.1.5)
     builder (3.2.4)
     concurrent-ruby (1.2.2)
+    connection_pool (2.4.1)
     crass (1.0.6)
-    debug (1.7.2)
-      irb (>= 1.5.0)
-      reline (>= 0.3.1)
+    debug (1.9.1)
+      irb (~> 1.10)
+      reline (>= 0.3.8)
     dotenv (2.8.1)
+    drb (2.2.0)
+      ruby2_keywords
     ed25519 (1.3.0)
     erubi (1.12.0)
-    i18n (1.12.0)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    io-console (0.6.0)
-    irb (1.6.3)
-      reline (>= 0.3.0)
-    loofah (2.20.0)
+    io-console (0.7.1)
+    irb (1.11.0)
+      rdoc
+      reline (>= 0.3.8)
+    loofah (2.22.0)
       crass (~> 1.0.2)
-      nokogiri (>= 1.5.9)
-    method_source (1.0.0)
-    minitest (5.18.0)
-    mocha (2.0.2)
+      nokogiri (>= 1.12.0)
+    minitest (5.20.0)
+    mocha (2.1.0)
       ruby2_keywords (>= 0.0.5)
+    mutex_m (0.2.0)
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
-    net-ssh (7.1.0)
-    nokogiri (1.14.2-arm64-darwin)
+    net-ssh (7.2.1)
+    nokogiri (1.16.0-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.2-x86_64-darwin)
+    nokogiri (1.16.0-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.2-x86_64-linux)
+    nokogiri (1.16.0-x86_64-linux)
       racc (~> 1.4)
-    racc (1.6.2)
-    rack (2.2.6.4)
+    psych (5.1.2)
+      stringio
+    racc (1.7.3)
+    rack (3.0.8)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
     rack-test (2.1.0)
       rack (>= 1.3)
-    rails-dom-testing (2.0.3)
-      activesupport (>= 4.2.0)
+    rackup (2.1.0)
+      rack (>= 3)
+      webrick (~> 1.8)
+    rails-dom-testing (2.2.0)
+      activesupport (>= 5.0.0)
+      minitest
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.5.0)
-      loofah (~> 2.19, >= 2.19.1)
-    railties (7.0.4.3)
-      actionpack (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      method_source
+    rails-html-sanitizer (1.6.0)
+      loofah (~> 2.21)
+      nokogiri (~> 1.14)
+    railties (7.1.2)
+      actionpack (= 7.1.2)
+      activesupport (= 7.1.2)
+      irb
+      rackup (>= 1.0.0)
       rake (>= 12.2)
-      thor (~> 1.0)
-      zeitwerk (~> 2.5)
-    rake (13.0.6)
-    reline (0.3.3)
+      thor (~> 1.0, >= 1.2.2)
+      zeitwerk (~> 2.6)
+    rake (13.1.0)
+    rdoc (6.6.2)
+      psych (>= 4.0.0)
+    reline (0.4.2)
       io-console (~> 0.5)
     ruby2_keywords (0.0.5)
-    sshkit (1.21.4)
+    sshkit (1.21.7)
+      mutex_m
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
-    thor (1.2.1)
+    stringio (3.1.0)
+    thor (1.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.7)
+    webrick (1.8.1)
+    zeitwerk (2.6.12)
 
 PLATFORMS
   arm64-darwin

README.md

@@ -1,6 +1,6 @@
 # Kamal: Deploy web apps anywhere
 
-From bare metal to cloud VMs using Docker, deploy web apps anywhere with zero downtime. Kamal uses the dynamic reverse-proxy Traefik to hold requests, while the new app container is started and the old one is stopped — working seamlessly across multiple hosts, using SSHKit to execute commands. Originally built for Rails apps, Kamal will work with any type of web app that can be containerized with Docker.
+From bare metal to cloud VMs, deploy web apps anywhere with zero downtime. Kamal has the dynamic reverse-proxy Traefik hold requests while a new app container is started and the old one is stopped. Works seamlessly across multiple hosts, using SSHKit to execute commands. Originally built for Rails apps, Kamal will work with any type of web app that can be containerized with Docker.
 
 ➡️ See [kamal-deploy.org](https://kamal-deploy.org) for documentation on [installation](https://kamal-deploy.org/docs/installation), [configuration](https://kamal-deploy.org/docs/configuration), and [commands](https://kamal-deploy.org/docs/commands).
 

gemfiles/ruby_2.7.gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+gemspec path: "../"
+
+gem "nokogiri", "~> 1.15.0"

kamal.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "ed25519", "~> 1.2"
   spec.add_dependency "bcrypt_pbkdf", "~> 1.0"
   spec.add_dependency "concurrent-ruby", "~> 1.2"
+  spec.add_dependency "base64", "~> 0.2"
 
   spec.add_development_dependency "debug"
   spec.add_development_dependency "mocha"

lib/kamal/cli/accessory.rb

@@ -5,11 +5,11 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
       if name == "all"
         KAMAL.accessory_names.each { |accessory_name| boot(accessory_name) }
       else
-        with_accessory(name) do |accessory|
+        with_accessory(name) do |accessory, hosts|
           directories(name)
           upload(name)
 
-          on(accessory.hosts) do
+          on(hosts) do
             execute *KAMAL.registry.login if login
             execute *KAMAL.auditor.record("Booted #{name} accessory"), verbosity: :debug
             execute *accessory.run
@@ -22,8 +22,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "upload [NAME]", "Upload accessory files to host", hide: true
   def upload(name)
     mutating do
-      with_accessory(name) do |accessory|
-        on(accessory.hosts) do
+      with_accessory(name) do |accessory, hosts|
+        on(hosts) do
           accessory.files.each do |(local, remote)|
             accessory.ensure_local_file_present(local)
 
@@ -39,8 +39,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "directories [NAME]", "Create accessory directories on host", hide: true
   def directories(name)
     mutating do
-      with_accessory(name) do |accessory|
-        on(accessory.hosts) do
+      with_accessory(name) do |accessory, hosts|
+        on(hosts) do
           accessory.directories.keys.each do |host_path|
             execute *accessory.make_directory(host_path)
           end
@@ -49,17 +49,21 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     end
   end
 
-  desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container)"
+  desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container; use NAME=all to boot all accessories)"
   def reboot(name)
     mutating do
-      with_accessory(name) do |accessory|
-        on(accessory.hosts) do
-          execute *KAMAL.registry.login
-        end
+      if name == "all"
+        KAMAL.accessory_names.each { |accessory_name| reboot(accessory_name) }
+      else
+        with_accessory(name) do |accessory, hosts|
+          on(hosts) do
+            execute *KAMAL.registry.login
+          end
 
-        stop(name)
-        remove_container(name)
-        boot(name, login: false)
+          stop(name)
+          remove_container(name)
+          boot(name, login: false)
+        end
       end
     end
   end
@@ -67,8 +71,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "start [NAME]", "Start existing accessory container on host"
   def start(name)
     mutating do
-      with_accessory(name) do |accessory|
-        on(accessory.hosts) do
+      with_accessory(name) do |accessory, hosts|
+        on(hosts) do
           execute *KAMAL.auditor.record("Started #{name} accessory"), verbosity: :debug
           execute *accessory.start
         end
@@ -79,8 +83,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "stop [NAME]", "Stop existing accessory container on host"
   def stop(name)
     mutating do
-      with_accessory(name) do |accessory|
-        on(accessory.hosts) do
+      with_accessory(name) do |accessory, hosts|
+        on(hosts) do
           execute *KAMAL.auditor.record("Stopped #{name} accessory"), verbosity: :debug
           execute *accessory.stop, raise_on_non_zero_exit: false
         end
@@ -103,8 +107,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     if name == "all"
       KAMAL.accessory_names.each { |accessory_name| details(accessory_name) }
     else
-      with_accessory(name) do |accessory|
-        on(accessory.hosts) { puts capture_with_info(*accessory.info) }
+      with_accessory(name) do |accessory, hosts|
+        on(hosts) { puts capture_with_info(*accessory.info) }
       end
     end
   end
@@ -113,7 +117,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
   option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
   def exec(name, cmd)
-    with_accessory(name) do |accessory|
+    with_accessory(name) do |accessory, hosts|
       case
       when options[:interactive] && options[:reuse]
         say "Launching interactive command with via SSH from existing container...", :magenta
@@ -125,14 +129,14 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
       when options[:reuse]
         say "Launching command from existing container...", :magenta
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
           capture_with_info(*accessory.execute_in_existing_container(cmd))
         end
 
       else
         say "Launching command from new container...", :magenta
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
           capture_with_info(*accessory.execute_in_new_container(cmd))
         end
@@ -146,12 +150,12 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
   option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
   def logs(name)
-    with_accessory(name) do |accessory|
+    with_accessory(name) do |accessory, hosts|
       grep = options[:grep]
 
       if options[:follow]
         run_locally do
-          info "Following logs on #{accessory.hosts}..."
+          info "Following logs on #{hosts}..."
           info accessory.follow_logs(grep: grep)
           exec accessory.follow_logs(grep: grep)
         end
@@ -159,7 +163,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
         since = options[:since]
         lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
-        on(accessory.hosts) do
+        on(hosts) do
           puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep))
         end
       end
@@ -188,8 +192,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "remove_container [NAME]", "Remove accessory container from host", hide: true
   def remove_container(name)
     mutating do
-      with_accessory(name) do |accessory|
-        on(accessory.hosts) do
+      with_accessory(name) do |accessory, hosts|
+        on(hosts) do
           execute *KAMAL.auditor.record("Remove #{name} accessory container"), verbosity: :debug
           execute *accessory.remove_container
         end
@@ -200,8 +204,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "remove_image [NAME]", "Remove accessory image from host", hide: true
   def remove_image(name)
     mutating do
-      with_accessory(name) do |accessory|
-        on(accessory.hosts) do
+      with_accessory(name) do |accessory, hosts|
+        on(hosts) do
           execute *KAMAL.auditor.record("Removed #{name} accessory image"), verbosity: :debug
           execute *accessory.remove_image
         end
@@ -212,8 +216,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "remove_service_directory [NAME]", "Remove accessory directory used for uploaded files and data directories from host", hide: true
   def remove_service_directory(name)
     mutating do
-      with_accessory(name) do |accessory|
-        on(accessory.hosts) do
+      with_accessory(name) do |accessory, hosts|
+        on(hosts) do
           execute *accessory.remove_service_directory
         end
       end
@@ -223,7 +227,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   private
     def with_accessory(name)
       if accessory = KAMAL.accessory(name)
-        yield accessory
+        yield accessory, accessory_hosts(accessory)
       else
         error_on_missing_accessory(name)
       end
@@ -236,4 +240,12 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
         "No accessory by the name of '#{name}'" +
         (options ? " (options: #{options.to_sentence})" : "")
     end
+
+    def accessory_hosts(accessory)
+      if KAMAL.specific_hosts&.any?
+        KAMAL.specific_hosts & accessory.hosts
+      else
+        accessory.hosts
+      end
+    end
 end

lib/kamal/cli/app.rb

@@ -9,19 +9,23 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
           on(KAMAL.hosts) do
             execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
-            execute *KAMAL.app.tag_current_as_latest
+            execute *KAMAL.app.tag_current_image_as_latest
+
+            KAMAL.roles_on(host).each do |role|
+              app = KAMAL.app(role: role)
+
+              if role.assets?
+                execute *app.extract_assets
+                old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+                execute *app.sync_asset_volumes(old_version: old_version)
+              end
+            end
           end
 
           on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
-            roles = KAMAL.roles_on(host)
-
-            roles.each do |role|
+            KAMAL.roles_on(host).each do |role|
               app = KAMAL.app(role: role)
               auditor = KAMAL.auditor(role: role)
-              role_config = KAMAL.config.role(role)
-
-              execute *app.extract_assets if role_config.assets?
-
 
               if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
                 tmp_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
@@ -31,30 +35,27 @@ class Kamal::Cli::App < Kamal::Cli::Base
               end
 
               old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-              original_old_version = old_version.gsub(/_replaced_[a-f0-9]{16}$/, "")
 
-              execute *app.sync_asset_volumes(old_version: original_old_version) if role_config.assets?
-
-              execute *app.tie_cord(role_config.cord_host_file) if role_config.uses_cord?
+              execute *app.tie_cord(role.cord_host_file) if role.uses_cord?
 
               execute *auditor.record("Booted app version #{version}"), verbosity: :debug
 
               execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
 
-              Kamal::Utils::HealthcheckPoller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+              Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
 
               if old_version.present?
-                if role_config.uses_cord?
+                if role.uses_cord?
                   cord = capture_with_info(*app.cord(version: old_version), raise_on_non_zero_exit: false).strip
                   if cord.present?
                     execute *app.cut_cord(cord)
-                    Kamal::Utils::HealthcheckPoller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
+                    Kamal::Cli::Healthcheck::Poller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
                   end
                 end
 
                 execute *app.stop(version: old_version), raise_on_non_zero_exit: false
 
-                execute *app.cleanup_assets if role_config.assets?
+                execute *app.clean_up_assets if role.assets?
               end
             end
           end
@@ -112,14 +113,16 @@ class Kamal::Cli::App < Kamal::Cli::Base
       say "Get current version of running container...", :magenta unless options[:version]
       using_version(options[:version] || current_running_version) do |version|
         say "Launching interactive command with version #{version} via SSH from existing container on #{KAMAL.primary_host}...", :magenta
-        run_locally { exec KAMAL.app(role: "web").execute_in_existing_container_over_ssh(cmd, host: KAMAL.primary_host) }
+        run_locally { exec KAMAL.app(role: KAMAL.primary_role).execute_in_existing_container_over_ssh(cmd, host: KAMAL.primary_host) }
       end
 
     when options[:interactive]
       say "Get most recent version available as an image...", :magenta unless options[:version]
       using_version(version_or_latest) do |version|
         say "Launching interactive command with version #{version} via SSH from new container on #{KAMAL.primary_host}...", :magenta
-        run_locally { exec KAMAL.app(role: KAMAL.roles_on(KAMAL.primary_host).first).execute_in_new_container_over_ssh(cmd, host: KAMAL.primary_host) }
+        run_locally do
+          exec KAMAL.app(role: KAMAL.primary_role).execute_in_new_container_over_ssh(cmd, host: KAMAL.primary_host)
+        end
       end
 
     when options[:reuse]
@@ -142,8 +145,12 @@ class Kamal::Cli::App < Kamal::Cli::Base
       using_version(version_or_latest) do |version|
         say "Launching command with version #{version} from new container...", :magenta
         on(KAMAL.hosts) do |host|
-          execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
-          puts_by_host host, capture_with_info(*KAMAL.app.execute_in_new_container(cmd))
+          roles = KAMAL.roles_on(host)
+
+          roles.each do |role|
+            execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_new_container(cmd))
+          end
         end
       end
     end
@@ -193,19 +200,20 @@ class Kamal::Cli::App < Kamal::Cli::Base
     # FIXME: Catch when app containers aren't running
 
     grep = options[:grep]
-
+    since = options[:since]
     if options[:follow]
+      lines = options[:lines].presence || ((since || grep) ? nil : 10) # Default to 10 lines if since or grep isn't set
+
       run_locally do
         info "Following logs on #{KAMAL.primary_host}..."
 
         KAMAL.specific_roles ||= ["web"]
         role = KAMAL.roles_on(KAMAL.primary_host).first
 
-        info KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, grep: grep)
-        exec KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, grep: grep)
+        info KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
+        exec KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
       end
     else
-      since = options[:since]
       lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
       on(KAMAL.hosts) do |host|

lib/kamal/cli/base.rb

@@ -14,8 +14,8 @@ module Kamal::Cli
     class_option :version, desc: "Run commands against a specific app version"
 
     class_option :primary, type: :boolean, aliases: "-p", desc: "Run commands only on primary host instead of all"
-    class_option :hosts, aliases: "-h", desc: "Run commands on these hosts instead of all (separate by comma)"
-    class_option :roles, aliases: "-r", desc: "Run commands on these roles instead of all (separate by comma)"
+    class_option :hosts, aliases: "-h", desc: "Run commands on these hosts instead of all (separate by comma, supports wildcards with *)"
+    class_option :roles, aliases: "-r", desc: "Run commands on these roles instead of all (separate by comma, supports wildcards with *)"
 
     class_option :config_file, aliases: "-c", default: "config/deploy.yml", desc: "Path to config file"
     class_option :destination, aliases: "-d", desc: "Specify destination to be used for config file (staging -> deploy.staging.yml)"
@@ -24,6 +24,7 @@ module Kamal::Cli
 
     def initialize(*)
       super
+      @original_env = ENV.to_h.dup
       load_envs
       initialize_commander(options_with_subcommand_class_options)
     end
@@ -37,6 +38,12 @@ module Kamal::Cli
         end
       end
 
+      def reload_envs
+        ENV.clear
+        ENV.update(@original_env)
+        load_envs
+      end
+
       def options_with_subcommand_class_options
         options.merge(@_initializer.last[:class_options] || {})
       end
@@ -75,8 +82,6 @@ module Kamal::Cli
       def mutating
         return yield if KAMAL.holding_lock?
 
-        KAMAL.config.ensure_env_available
-
         run_hook "pre-connect"
 
         ensure_run_directory
@@ -118,8 +123,9 @@ module Kamal::Cli
         yield
       rescue SSHKit::Runner::ExecuteError => e
         if e.message =~ /cannot create directory/
+          say "Deploy lock already in place!", :red
           on(KAMAL.primary_host) { puts capture_with_debug(*KAMAL.lock.status) }
-          raise LockError, "Deploy lock found"
+          raise LockError, "Deploy lock found. Run 'kamal lock help' for more information"
         else
           raise e
         end

lib/kamal/cli/build.rb

@@ -1,3 +1,5 @@
+require "uri"
+
 class Kamal::Cli::Build < Kamal::Cli::Base
   class BuildError < StandardError; end
 
@@ -17,7 +19,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base
       verify_local_dependencies
       run_hook "pre-build"
 
-      if (uncommitted_changes = Kamal::Utils.uncommitted_changes).present?
+      if (uncommitted_changes = Kamal::Git.uncommitted_changes).present?
         say "The following paths have uncommitted changes:\n #{uncommitted_changes}", :yellow
       end
 
@@ -48,6 +50,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base
         execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
         execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
         execute *KAMAL.builder.pull
+        execute *KAMAL.builder.validate_image
       end
     end
   end
@@ -55,6 +58,10 @@ class Kamal::Cli::Build < Kamal::Cli::Base
   desc "create", "Create a build setup"
   def create
     mutating do
+      if (remote_host = KAMAL.config.builder.remote_host)
+        connect_to_remote_host(remote_host)
+      end
+
       run_locally do
         begin
           debug "Using builder: #{KAMAL.builder.name}"
@@ -103,4 +110,14 @@ class Kamal::Cli::Build < Kamal::Cli::Base
         end
       end
     end
+
+    def connect_to_remote_host(remote_host)
+      remote_uri = URI.parse(remote_host)
+      if remote_uri.scheme == "ssh"
+        options = { user: remote_uri.user, port: remote_uri.port }.compact
+        on(remote_uri.host, options) do
+          execute "true"
+        end
+      end
+    end
 end

lib/kamal/cli/env.rb

@@ -5,10 +5,11 @@ class Kamal::Cli::Env < Kamal::Cli::Base
   def push
     mutating do
       on(KAMAL.hosts) do
+        execute *KAMAL.auditor.record("Pushed env files"), verbosity: :debug
+
         KAMAL.roles_on(host).each do |role|
-          role_config = KAMAL.config.role(role)
           execute *KAMAL.app(role: role).make_env_directory
-          upload! StringIO.new(role_config.env_file), role_config.host_env_file_path, mode: 400
+          upload! StringIO.new(role.env_file), role.host_env_file_path, mode: 400
         end
       end
 
@@ -31,8 +32,9 @@ class Kamal::Cli::Env < Kamal::Cli::Base
   def delete
     mutating do
       on(KAMAL.hosts) do
+        execute *KAMAL.auditor.record("Deleted env files"), verbosity: :debug
+
         KAMAL.roles_on(host).each do |role|
-          role_config = KAMAL.config.role(role)
           execute *KAMAL.app(role: role).remove_env_file
         end
       end

lib/kamal/cli/healthcheck.rb

@@ -3,11 +3,12 @@ class Kamal::Cli::Healthcheck < Kamal::Cli::Base
 
   desc "perform", "Health check current app version"
   def perform
+    raise "The primary host is not configured to run Traefik" unless KAMAL.config.role(KAMAL.config.primary_role).running_traefik?
     on(KAMAL.primary_host) do
       begin
         execute *KAMAL.healthcheck.run
-        Kamal::Utils::HealthcheckPoller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
-      rescue Kamal::Utils::HealthcheckPoller::HealthcheckError => e
+        Poller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
+      rescue Poller::HealthcheckError => e
         error capture_with_info(*KAMAL.healthcheck.logs)
         error capture_with_pretty_json(*KAMAL.healthcheck.container_health_log)
         raise

lib/kamal/cli/healthcheck/poller.rb

@@ -0,0 +1,64 @@
+module Kamal::Cli::Healthcheck::Poller
+  extend self
+
+  TRAEFIK_UPDATE_DELAY = 5
+
+  class HealthcheckError < StandardError; end
+
+  def wait_for_healthy(pause_after_ready: false, &block)
+    attempt = 1
+    max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+    begin
+      case status = block.call
+      when "healthy"
+        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
+      when "running" # No health check configured
+        sleep KAMAL.config.readiness_delay if pause_after_ready
+      else
+        raise HealthcheckError, "container not ready (#{status})"
+      end
+    rescue HealthcheckError => e
+      if attempt <= max_attempts
+        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+        sleep attempt
+        attempt += 1
+        retry
+      else
+        raise
+      end
+    end
+
+    info "Container is healthy!"
+  end
+
+  def wait_for_unhealthy(pause_after_ready: false, &block)
+    attempt = 1
+    max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+    begin
+      case status = block.call
+      when "unhealthy"
+        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
+      else
+        raise HealthcheckError, "container not unhealthy (#{status})"
+      end
+    rescue HealthcheckError => e
+      if attempt <= max_attempts
+        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+        sleep attempt
+        attempt += 1
+        retry
+      else
+        raise
+      end
+    end
+
+    info "Container is unhealthy!"
+  end
+
+  private
+    def info(message)
+      SSHKit.config.output.info(message)
+    end
+end

lib/kamal/cli/main.rb

@@ -1,10 +1,18 @@
 class Kamal::Cli::Main < Kamal::Cli::Base
-  desc "setup", "Setup all accessories and deploy app to servers"
+  desc "setup", "Setup all accessories, push the env, and deploy app to servers"
+  option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def setup
     print_runtime do
       mutating do
-        invoke "kamal:cli:server:bootstrap"
-        invoke "kamal:cli:accessory:boot", [ "all" ]
+        invoke_options = deploy_options
+
+        say "Ensure Docker is installed...", :magenta
+        invoke "kamal:cli:server:bootstrap", [], invoke_options
+
+        say "Push env files...", :magenta
+        invoke "kamal:cli:env:push", [], invoke_options
+
+        invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options
         deploy
       end
     end
@@ -33,11 +41,13 @@ class Kamal::Cli::Main < Kamal::Cli::Base
         say "Ensure Traefik is running...", :magenta
         invoke "kamal:cli:traefik:boot", [], invoke_options
 
-        say "Ensure app can pass healthcheck...", :magenta
-        invoke "kamal:cli:healthcheck:perform", [], invoke_options
+        if KAMAL.config.role(KAMAL.config.primary_role).running_traefik?
+          say "Ensure app can pass healthcheck...", :magenta
+          invoke "kamal:cli:healthcheck:perform", [], invoke_options
+        end
 
         say "Detect stale containers...", :magenta
-        invoke "kamal:cli:app:stale_containers", [], invoke_options
+        invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
 
         invoke "kamal:cli:app:boot", [], invoke_options
 
@@ -70,7 +80,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
         invoke "kamal:cli:healthcheck:perform", [], invoke_options
 
         say "Detect stale containers...", :magenta
-        invoke "kamal:cli:app:stale_containers", [], invoke_options
+        invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
 
         invoke "kamal:cli:app:boot", [], invoke_options
       end
@@ -165,6 +175,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   end
 
   desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)"
+  option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip .env file push"
   def envify
     if destination = options[:destination]
       env_template_path = ".env.#{destination}.erb"
@@ -174,10 +185,12 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       env_path          = ".env"
     end
 
-    File.write(env_path, ERB.new(File.read(env_template_path)).result, perm: 0600)
+    File.write(env_path, ERB.new(File.read(env_template_path), trim_mode: "-").result, perm: 0600)
 
-    load_envs # reload new file
-    invoke "kamal:cli:env:push", options
+    unless options[:skip_push]
+      reload_envs
+      invoke "kamal:cli:env:push", options
+    end
   end
 
   desc "remove", "Remove Traefik, app, accessories, and registry session from servers"

lib/kamal/cli/prune.rb

@@ -7,7 +7,7 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
     end
   end
 
-  desc "images", "Prune dangling images"
+  desc "images", "Prune unused images"
   def images
     mutating do
       on(KAMAL.hosts) do
@@ -18,12 +18,17 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
     end
   end
 
-  desc "containers", "Prune all stopped containers, except the last 5"
+  desc "containers", "Prune all stopped containers, except the last n (default 5)"
+  option :retain, type: :numeric, default: nil, desc: "Number of containers to retain"
   def containers
+    retain = options.fetch(:retain, KAMAL.config.retain_containers)
+    raise "retain must be at least 1" if retain < 1
+
     mutating do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug
-        execute *KAMAL.prune.containers
+        execute *KAMAL.prune.app_containers(retain: retain)
+        execute *KAMAL.prune.healthcheck_containers
       end
     end
   end

lib/kamal/cli/server.rb

@@ -12,14 +12,14 @@ class Kamal::Cli::Server < Kamal::Cli::Base
           missing << host
         end
       end
-    end
 
-    on(KAMAL.hosts) do
       execute(*KAMAL.server.ensure_run_directory)
     end
 
     if missing.any?
-      raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/"
+      raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and either `wget` or `curl`. Install Docker manually: https://docs.docker.com/engine/install/"
     end
+
+    run_hook "docker-setup"
   end
 end

lib/kamal/cli/templates/deploy.yml

@@ -19,6 +19,7 @@ registry:
     - KAMAL_REGISTRY_PASSWORD
 
 # Inject ENV variables into containers (secrets come from .env).
+# Remember to run `kamal env push` after making changes!
 # env:
 #   clear:
 #     DB_HOST: 192.168.0.2
@@ -52,7 +53,7 @@ registry:
 #         - MYSQL_ROOT_PASSWORD
 #     files:
 #       - config/mysql/production.cnf:/etc/mysql/my.cnf
-#       - db/production.sql.erb:/docker-entrypoint-initdb.d/setup.sql
+#       - db/production.sql:/docker-entrypoint-initdb.d/setup.sql
 #     directories:
 #       - data:/var/lib/mysql
 #   redis:
@@ -72,3 +73,29 @@ registry:
 # healthcheck:
 #   path: /healthz
 #   port: 4000
+
+# Bridge fingerprinted assets, like JS and CSS, between versions to avoid
+# hitting 404 on in-flight requests. Combines all files from new and old
+# version inside the asset_path.
+#
+# If your app is using the Sprockets gem, ensure it sets `config.assets.manifest`.
+# See https://github.com/basecamp/kamal/issues/626 for details
+#
+# asset_path: /rails/public/assets
+
+# Configure rolling deploys by setting a wait time between batches of restarts.
+# boot:
+#   limit: 10 # Can also specify as a percentage of total hosts, such as "25%"
+#   wait: 2
+
+# Configure the role used to determine the primary_host. This host takes
+# deploy locks, runs health checks during the deploy, and follow logs, etc.
+#
+# Caution: there's no support for role renaming yet, so be careful to cleanup
+#          the previous role on the deployed hosts.
+# primary_role: web
+
+# Controls if we abort when see a role with no hosts. Disabling this may be
+# useful for more complex deploy configurations.
+#
+# allow_empty_roles: false

lib/kamal/cli/templates/sample_hooks/docker-setup.sample

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+# A sample docker-setup hook
+#
+# Sets up a Docker network which can then be used by the application’s containers
+
+ssh user@example.com docker network create kamal

lib/kamal/cli/templates/sample_hooks/post-traefik-reboot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooted Traefik on $KAMAL_HOSTS"

lib/kamal/cli/templates/sample_hooks/pre-build.sample

@@ -32,7 +32,7 @@ fi
 current_branch=$(git branch --show-current)
 
 if [ -z "$current_branch" ]; then
-  echo "No git remote set, aborting..." >&2
+  echo "Not on a git branch, aborting..." >&2
   exit 1
 fi
 

lib/kamal/cli/templates/sample_hooks/pre-traefik-reboot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooting Traefik on $KAMAL_HOSTS..."

lib/kamal/cli/traefik.rb

@@ -13,12 +13,18 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
   option :rolling, type: :boolean, default: false, desc: "Reboot traefik on hosts in sequence, rather than in parallel"
   def reboot
     mutating do
-      on(KAMAL.traefik_hosts, in: options[:rolling] ? :sequence : :parallel) do
-        execute *KAMAL.auditor.record("Rebooted traefik"), verbosity: :debug
-        execute *KAMAL.registry.login
-        execute *KAMAL.traefik.stop
-        execute *KAMAL.traefik.remove_container
-        execute *KAMAL.traefik.run
+      host_groups = options[:rolling] ? KAMAL.traefik_hosts : [KAMAL.traefik_hosts]
+      host_groups.each do |hosts|
+        host_list = Array(hosts).join(",")
+        run_hook "pre-traefik-reboot", hosts: host_list
+        on(hosts) do
+          execute *KAMAL.auditor.record("Rebooted traefik"), verbosity: :debug
+          execute *KAMAL.registry.login
+          execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
+          execute *KAMAL.traefik.remove_container
+          execute *KAMAL.traefik.run
+        end
+        run_hook "post-traefik-reboot", hosts: host_list
       end
     end
   end
@@ -38,7 +44,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
     mutating do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.auditor.record("Stopped traefik"), verbosity: :debug
-        execute *KAMAL.traefik.stop
+        execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
       end
     end
   end

lib/kamal/commander.rb

@@ -24,19 +24,40 @@ class Kamal::Commander
   attr_reader :specific_roles, :specific_hosts
 
   def specific_primary!
-    self.specific_hosts = [ config.primary_web_host ]
+    self.specific_hosts = [ config.primary_host ]
   end
 
   def specific_roles=(role_names)
-    @specific_roles = config.roles.select { |r| role_names.include?(r.name) } if role_names.present?
+    if role_names.present?
+      @specific_roles = Kamal::Utils.filter_specific_items(role_names, config.roles)
+
+      if @specific_roles.empty?
+        raise ArgumentError, "No --roles match for #{role_names.join(',')}"
+      end
+
+      @specific_roles
+    end
   end
 
   def specific_hosts=(hosts)
-    @specific_hosts = config.all_hosts & hosts if hosts.present?
+    if hosts.present?
+      @specific_hosts = Kamal::Utils.filter_specific_items(hosts, config.all_hosts)
+
+      if @specific_hosts.empty?
+        raise ArgumentError, "No --hosts match for #{hosts.join(',')}"
+      end
+
+      @specific_hosts
+    end
   end
 
   def primary_host
-    specific_hosts&.first || specific_roles&.first&.primary_host || config.primary_web_host
+    # Given a list of specific roles, make an effort to match up with the primary_role
+    specific_hosts&.first || specific_roles&.detect { |role| role == config.primary_role }&.primary_host || specific_roles&.first&.primary_host || config.primary_host
+  end
+
+  def primary_role
+    roles_on(primary_host).first
   end
 
   def roles
@@ -51,16 +72,8 @@ class Kamal::Commander
     end
   end
 
-  def boot_strategy
-    if config.boot.limit.present?
-      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
-    else
-      {}
-    end
-  end
-
   def roles_on(host)
-    roles.select { |role| role.hosts.include?(host.to_s) }.map(&:name)
+    roles.select { |role| role.hosts.include?(host.to_s) }
   end
 
   def traefik_hosts
@@ -128,6 +141,7 @@ class Kamal::Commander
     @traefik ||= Kamal::Commands::Traefik.new(config)
   end
 
+
   def with_verbosity(level)
     old_level = self.verbosity
 
@@ -140,6 +154,14 @@ class Kamal::Commander
     SSHKit.config.output_verbosity = old_level
   end
 
+  def boot_strategy
+    if config.boot.limit.present?
+      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
+    else
+      {}
+    end
+  end
+
   def holding_lock?
     self.holding_lock
   end

lib/kamal/commands/app.rb

@@ -1,12 +1,13 @@
 class Kamal::Commands::App < Kamal::Commands::Base
+  include Assets, Containers, Cord, Execution, Images, Logging
+
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 
-  attr_reader :role, :role_config
+  attr_reader :role, :role
 
   def initialize(config, role: nil)
     super(config)
     @role = role
-    @role_config = config.role(self.role)
   end
 
   def run(hostname: nil)
@@ -16,15 +17,16 @@ class Kamal::Commands::App < Kamal::Commands::Base
       "--name", container_name,
       *(["--hostname", hostname] if hostname),
       "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
-      *role_config.env_args,
-      *role_config.health_check_args,
-      *config.logging_args,
+      "-e", "KAMAL_VERSION=\"#{config.version}\"",
+      *role.env_args,
+      *role.health_check_args,
+      *role.logging_args,
       *config.volume_args,
-      *role_config.asset_volume_args,
-      *role_config.label_args,
-      *role_config.option_args,
+      *role.asset_volume_args,
+      *role.label_args,
+      *role.option_args,
       config.absolute_image,
-      role_config.cmd
+      role.cmd
   end
 
   def start
@@ -46,51 +48,6 @@ class Kamal::Commands::App < Kamal::Commands::Base
   end
 
 
-  def logs(since: nil, lines: nil, grep: nil)
-    pipe \
-      current_running_container_id,
-      "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
-      ("grep '#{grep}'" if grep)
-  end
-
-  def follow_logs(host:, grep: nil)
-    run_over_ssh \
-      pipe(
-        current_running_container_id,
-        "xargs docker logs --timestamps --tail 10 --follow 2>&1",
-        (%(grep "#{grep}") if grep)
-      ),
-      host: host
-  end
-
-
-  def execute_in_existing_container(*command, interactive: false)
-    docker :exec,
-      ("-it" if interactive),
-      container_name,
-      *command
-  end
-
-  def execute_in_new_container(*command, interactive: false)
-    docker :run,
-      ("-it" if interactive),
-      "--rm",
-      *role_config&.env_args,
-      *config.volume_args,
-      *role_config&.option_args,
-      config.absolute_image,
-      *command
-  end
-
-  def execute_in_existing_container_over_ssh(*command, host:)
-    run_over_ssh execute_in_existing_container(*command, interactive: true), host: host
-  end
-
-  def execute_in_new_container_over_ssh(*command, host:)
-    run_over_ssh execute_in_new_container(*command, interactive: true), host: host
-  end
-
-
   def current_running_container_id
     docker :ps, "--quiet", *filter_args(statuses: ACTIVE_DOCKER_STATUSES), "--latest"
   end
@@ -106,102 +63,22 @@ class Kamal::Commands::App < Kamal::Commands::Base
   def list_versions(*docker_args, statuses: nil)
     pipe \
       docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
-      %(while read line; do echo ${line##{role_config.container_prefix}-}; done) # Extract SHA from "service-role-dest-SHA"
+      %(while read line; do echo ${line##{role.container_prefix}-}; done) # Extract SHA from "service-role-dest-SHA"
   end
 
-  def list_containers
-    docker :container, :ls, "--all", *filter_args
-  end
-
-  def list_container_names
-    [ *list_containers, "--format", "'{{ .Names }}'" ]
-  end
-
-  def remove_container(version:)
-    pipe \
-      container_id_for(container_name: container_name(version)),
-      xargs(docker(:container, :rm))
-  end
-
-  def rename_container(version:, new_version:)
-    docker :rename, container_name(version), container_name(new_version)
-  end
-
-  def remove_containers
-    docker :container, :prune, "--force", *filter_args
-  end
-
-  def list_images
-    docker :image, :ls, config.repository
-  end
-
-  def remove_images
-    docker :image, :prune, "--all", "--force", *filter_args
-  end
-
-  def tag_current_as_latest
-    docker :tag, config.absolute_image, config.latest_image
-  end
 
   def make_env_directory
-    make_directory role_config.host_env_directory
+    make_directory role.host_env_directory
   end
 
   def remove_env_file
-    [:rm, "-f", role_config.host_env_file_path]
+    [ :rm, "-f", role.host_env_file_path ]
   end
 
-  def cord(version:)
-    pipe \
-      docker(:inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", container_name(version)),
-      [:awk, "'$2 == \"#{role_config.cord_volume.container_path}\" {print $1}'"]
-  end
-
-  def tie_cord(cord)
-    create_empty_file(cord)
-  end
-
-  def cut_cord(cord)
-    remove_directory(cord)
-  end
-
-  def extract_assets
-    asset_container = "#{role_config.container_prefix}-assets"
-
-    combine \
-      make_directory(role_config.asset_extracted_path),
-      [*docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true"],
-      docker(:run, "--name", asset_container, "--detach", "--rm", config.latest_image, "sleep infinity"),
-      docker(:cp, "-L", "#{asset_container}:#{role_config.asset_path}/.", role_config.asset_extracted_path),
-      docker(:stop, "-t 1", asset_container),
-      by: "&&"
-  end
-
-  def sync_asset_volumes(old_version: nil)
-    new_extracted_path, new_volume_path = role_config.asset_extracted_path(config.version), role_config.asset_volume.host_path
-    if old_version.present?
-      old_extracted_path, old_volume_path = role_config.asset_extracted_path(old_version), role_config.asset_volume(old_version).host_path
-    end
-
-    commands = [make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path)]
-
-    if old_version.present?
-      commands << copy_contents(new_extracted_path, old_volume_path)
-      commands << copy_contents(old_extracted_path, new_volume_path)
-    end
-
-    chain *commands
-  end
-
-  def cleanup_assets
-    chain \
-      find_and_remove_older_siblings(role_config.asset_extracted_path),
-      find_and_remove_older_siblings(role_config.asset_volume_path)
-  end
 
   private
     def container_name(version = nil)
-      [ role_config.container_prefix, version || config.version ].compact.join("-")
+      [ role.container_prefix, version || config.version ].compact.join("-")
     end
 
     def filter_args(statuses: nil)
@@ -209,7 +86,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
     end
 
     def service_role_dest
-      [config.service, role, config.destination].compact.join("-")
+      [ config.service, role, config.destination ].compact.join("-")
     end
 
     def filters(statuses: nil)
@@ -221,19 +98,4 @@ class Kamal::Commands::App < Kamal::Commands::Base
         end
       end
     end
-
-    def find_and_remove_older_siblings(path)
-      [
-        :find,
-        Pathname.new(path).dirname,
-        "-maxdepth 1",
-        "-name", "'#{role_config.container_prefix}-*'",
-        "!", "-name", Pathname.new(path).basename,
-        "-exec rm -rf \"{}\" +"
-      ]
-    end
-
-    def copy_contents(source, destination)
-      [ :cp, "-rn", "#{source}/*", destination ]
-    end
 end

lib/kamal/commands/app/assets.rb

@@ -0,0 +1,51 @@
+module Kamal::Commands::App::Assets
+  def extract_assets
+    asset_container = "#{role.container_prefix}-assets"
+
+    combine \
+      make_directory(role.asset_extracted_path),
+      [*docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true"],
+      docker(:run, "--name", asset_container, "--detach", "--rm", config.latest_image, "sleep 1000000"),
+      docker(:cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_path),
+      docker(:stop, "-t 1", asset_container),
+      by: "&&"
+  end
+
+  def sync_asset_volumes(old_version: nil)
+    new_extracted_path, new_volume_path = role.asset_extracted_path(config.version), role.asset_volume.host_path
+    if old_version.present?
+      old_extracted_path, old_volume_path = role.asset_extracted_path(old_version), role.asset_volume(old_version).host_path
+    end
+
+    commands = [make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path)]
+
+    if old_version.present?
+      commands << copy_contents(new_extracted_path, old_volume_path, continue_on_error: true)
+      commands << copy_contents(old_extracted_path, new_volume_path, continue_on_error: true)
+    end
+
+    chain *commands
+  end
+
+  def clean_up_assets
+    chain \
+      find_and_remove_older_siblings(role.asset_extracted_path),
+      find_and_remove_older_siblings(role.asset_volume_path)
+  end
+
+  private
+    def find_and_remove_older_siblings(path)
+      [
+        :find,
+        Pathname.new(path).dirname.to_s,
+        "-maxdepth 1",
+        "-name", "'#{role.container_prefix}-*'",
+        "!", "-name", Pathname.new(path).basename.to_s,
+        "-exec rm -rf \"{}\" +"
+      ]
+    end
+
+    def copy_contents(source, destination, continue_on_error: false)
+      [ :cp, "-rnT", "#{source}", destination, *("|| true" if continue_on_error)]
+    end
+end

lib/kamal/commands/app/containers.rb

@@ -0,0 +1,23 @@
+module Kamal::Commands::App::Containers
+  def list_containers
+    docker :container, :ls, "--all", *filter_args
+  end
+
+  def list_container_names
+    [ *list_containers, "--format", "'{{ .Names }}'" ]
+  end
+
+  def remove_container(version:)
+    pipe \
+      container_id_for(container_name: container_name(version)),
+      xargs(docker(:container, :rm))
+  end
+
+  def rename_container(version:, new_version:)
+    docker :rename, container_name(version), container_name(new_version)
+  end
+
+  def remove_containers
+    docker :container, :prune, "--force", *filter_args
+  end
+end

lib/kamal/commands/app/cord.rb

@@ -0,0 +1,22 @@
+module Kamal::Commands::App::Cord
+  def cord(version:)
+    pipe \
+      docker(:inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", container_name(version)),
+      [:awk, "'$2 == \"#{role.cord_volume.container_path}\" {print $1}'"]
+  end
+
+  def tie_cord(cord)
+    create_empty_file(cord)
+  end
+
+  def cut_cord(cord)
+    remove_directory(cord)
+  end
+
+  private
+    def create_empty_file(file)
+      chain \
+        make_directory_for(file),
+        [:touch, file]
+    end
+end

lib/kamal/commands/app/execution.rb

@@ -0,0 +1,27 @@
+module Kamal::Commands::App::Execution
+  def execute_in_existing_container(*command, interactive: false)
+    docker :exec,
+      ("-it" if interactive),
+      container_name,
+      *command
+  end
+
+  def execute_in_new_container(*command, interactive: false)
+    docker :run,
+      ("-it" if interactive),
+      "--rm",
+      *role&.env_args,
+      *config.volume_args,
+      *role&.option_args,
+      config.absolute_image,
+      *command
+  end
+
+  def execute_in_existing_container_over_ssh(*command, host:)
+    run_over_ssh execute_in_existing_container(*command, interactive: true), host: host
+  end
+
+  def execute_in_new_container_over_ssh(*command, host:)
+    run_over_ssh execute_in_new_container(*command, interactive: true), host: host
+  end
+end

lib/kamal/commands/app/images.rb

@@ -0,0 +1,13 @@
+module Kamal::Commands::App::Images
+  def list_images
+    docker :image, :ls, config.repository
+  end
+
+  def remove_images
+    docker :image, :prune, "--all", "--force", *filter_args
+  end
+
+  def tag_current_image_as_latest
+    docker :tag, config.absolute_image, config.latest_image
+  end
+end

lib/kamal/commands/app/logging.rb

@@ -0,0 +1,18 @@
+module Kamal::Commands::App::Logging
+  def logs(since: nil, lines: nil, grep: nil)
+    pipe \
+      current_running_container_id,
+      "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
+      ("grep '#{grep}'" if grep)
+  end
+
+  def follow_logs(host:, lines: nil, grep: nil)
+    run_over_ssh \
+      pipe(
+        current_running_container_id,
+        "xargs docker logs --timestamps#{" --tail #{lines}" if lines} --follow 2>&1",
+        (%(grep "#{grep}") if grep)
+      ),
+      host: host
+  end
+end

lib/kamal/commands/base.rb

@@ -18,7 +18,7 @@ module Kamal::Commands
         elsif config.ssh.proxy && config.ssh.proxy.is_a?(Net::SSH::Proxy::Command)
           cmd << " -o ProxyCommand='#{config.ssh.proxy.command_line_template}'"
         end
-        cmd << " -t #{config.ssh.user}@#{host} '#{command.join(" ")}'"
+        cmd << " -t #{config.ssh.user}@#{host} -p #{config.ssh.port} '#{command.join(" ")}'"
       end
     end
 
@@ -62,10 +62,18 @@ module Kamal::Commands
         combine *commands, by: ">"
       end
 
+      def any(*commands)
+        combine *commands, by: "||"
+      end
+
       def xargs(command)
         [ :xargs, command ].flatten
       end
 
+      def shell(command)
+        [ :sh, "-c", "'#{command.flatten.join(" ").gsub("'", "'\\''")}'" ]
+      end
+
       def docker(*args)
         args.compact.unshift :docker
       end
@@ -73,11 +81,5 @@ module Kamal::Commands
       def tags(**details)
         Kamal::Tags.from_config(config, **details)
       end
-
-      def create_empty_file(file)
-        chain \
-          make_directory_for(file),
-          [:touch, file]
-      end
   end
 end

lib/kamal/commands/builder.rb

@@ -1,7 +1,7 @@
 require "active_support/core_ext/string/filters"
 
 class Kamal::Commands::Builder < Kamal::Commands::Base
-  delegate :create, :remove, :push, :clean, :pull, :info, to: :target
+  delegate :create, :remove, :push, :clean, :pull, :info, :validate_image, to: :target
 
   def name
     target.class.to_s.remove("Kamal::Commands::Builder::").underscore.inquiry

lib/kamal/commands/builder/base.rb

@@ -3,7 +3,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
   class BuilderError < StandardError; end
 
   delegate :argumentize, to: Kamal::Utils
-  delegate :args, :secrets, :dockerfile, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, to: :builder_config
+  delegate :args, :secrets, :dockerfile, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, :ssh, to: :builder_config
 
   def clean
     docker :image, :rm, "--force", config.absolute_image
@@ -14,13 +14,22 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
   end
 
   def build_options
-    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile ]
+    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_ssh ]
   end
 
   def build_context
     config.builder.context
   end
 
+  def validate_image
+    pipe \
+      docker(:inspect, "-f", "'{{ .Config.Labels.service }}'", config.absolute_image),
+      any(
+        [:grep, "-x", config.service],
+        "(echo \"Image #{config.absolute_image} is missing the 'service' label\" && exit 1)"
+      )
+  end
+
 
   private
     def build_tags
@@ -54,6 +63,10 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
       end
     end
 
+    def build_ssh
+      argumentize "--ssh", ssh if ssh.present?
+    end
+
     def builder_config
       config.builder
     end

lib/kamal/commands/builder/multiarch.rb

@@ -10,7 +10,7 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
   def push
     docker :buildx, :build,
       "--push",
-      "--platform", "linux/amd64,linux/arm64",
+      "--platform", platform_names,
       "--builder", builder_name,
       *build_options,
       build_context
@@ -26,4 +26,12 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
     def builder_name
       "kamal-#{config.service}-multiarch"
     end
+
+    def platform_names
+      if local_arch
+        "linux/#{local_arch}"
+      else
+        "linux/amd64,linux/arm64"
+      end
+    end
 end

lib/kamal/commands/docker.rb

@@ -1,7 +1,7 @@
 class Kamal::Commands::Docker < Kamal::Commands::Base
   # Install Docker using the https://github.com/docker/docker-install convenience script.
   def install
-    pipe [ :curl, "-fsSL", "https://get.docker.com" ], :sh
+    pipe get_docker, :sh
   end
 
   # Checks the Docker client version. Fails if Docker is not installed.
@@ -16,6 +16,15 @@ class Kamal::Commands::Docker < Kamal::Commands::Base
 
   # Do we have superuser access to install Docker and start system services?
   def superuser?
-    [ '[ "${EUID:-$(id -u)}" -eq 0 ]' ]
+    [ '[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null' ]
   end
+
+  private
+    def get_docker
+      shell \
+        any \
+          [ :curl, "-fsSL", "https://get.docker.com" ],
+          [ :wget, "-O -", "https://get.docker.com" ],
+          [ :echo, "\"exit 1\"" ]
+    end
 end

lib/kamal/commands/healthcheck.rb

@@ -1,20 +1,20 @@
 class Kamal::Commands::Healthcheck < Kamal::Commands::Base
 
   def run
-    web = config.role(:web)
+    primary = config.role(config.primary_role)
 
     docker :run,
       "--detach",
       "--name", container_name_with_version,
       "--publish", "#{exposed_port}:#{config.healthcheck["port"]}",
-      "--label", "service=#{container_name}",
-      "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
-      *web.env_args,
-      *web.health_check_args(cord: false),
+      "--label", "service=#{config.healthcheck_service}",
+      "-e", "KAMAL_CONTAINER_NAME=\"#{config.healthcheck_service}\"",
+      *primary.env_args,
+      *primary.health_check_args(cord: false),
       *config.volume_args,
-      *web.option_args,
+      *primary.option_args,
       config.absolute_image,
-      web.cmd
+      primary.cmd
   end
 
   def status
@@ -26,7 +26,7 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
   end
 
   def logs
-    pipe container_id, xargs(docker(:logs, "--tail", 50, "2>&1"))
+    pipe container_id, xargs(docker(:logs, "--tail", log_lines, "2>&1"))
   end
 
   def stop
@@ -38,12 +38,8 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
   end
 
   private
-    def container_name
-      [ "healthcheck", config.service, config.destination ].compact.join("-")
-    end
-
     def container_name_with_version
-      "#{container_name}-#{config.version}"
+      "#{config.healthcheck_service}-#{config.version}"
     end
 
     def container_id
@@ -57,4 +53,8 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
     def exposed_port
       config.healthcheck["exposed_port"]
     end
+
+    def log_lines
+      config.healthcheck["log_lines"]
+    end
 end

lib/kamal/commands/lock.rb

@@ -1,5 +1,6 @@
 require "active_support/duration"
 require "time"
+require "base64"
 
 class Kamal::Commands::Lock < Kamal::Commands::Base
   def acquire(message, version)
@@ -56,7 +57,7 @@ class Kamal::Commands::Lock < Kamal::Commands::Base
     end
 
     def locked_by
-      `git config user.name`.strip
+      Kamal::Git.user_name
     rescue Errno::ENOENT
       "Unknown"
     end

lib/kamal/commands/prune.rb

@@ -3,7 +3,7 @@ require "active_support/core_ext/numeric/time"
 
 class Kamal::Commands::Prune < Kamal::Commands::Base
   def dangling_images
-    docker :image, :prune, "--force", "--filter", "label=service=#{config.service}", "--filter", "dangling=true"
+    docker :image, :prune, "--force", "--filter", "label=service=#{config.service}"
   end
 
   def tagged_images
@@ -13,13 +13,17 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
       "while read image tag; do docker rmi $tag; done"
   end
 
-  def containers(keep_last: 5)
+  def app_containers(retain:)
     pipe \
       docker(:ps, "-q", "-a", *service_filter, *stopped_containers_filters),
-      "tail -n +#{keep_last + 1}",
+      "tail -n +#{retain + 1}",
       "while read container_id; do docker rm $container_id; done"
   end
 
+  def healthcheck_containers
+    docker :container, :prune, "--force", *healthcheck_service_filter
+  end
+
   private
     def stopped_containers_filters
       [ "created", "exited", "dead" ].flat_map { |status| ["--filter", "status=#{status}"] }
@@ -35,4 +39,8 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
     def service_filter
       [ "--filter", "label=service=#{config.service}" ]
     end
-end
+
+    def healthcheck_service_filter
+      [ "--filter", "label=service=#{config.healthcheck_service}" ]
+    end
+  end

lib/kamal/commands/registry.rb

@@ -2,7 +2,10 @@ class Kamal::Commands::Registry < Kamal::Commands::Base
   delegate :registry, to: :config
 
   def login
-    docker :login, registry["server"], "-u", sensitive(lookup("username")), "-p", sensitive(lookup("password"))
+    docker :login,
+      registry["server"],
+      "-u", sensitive(Kamal::Utils.escape_shell_value(lookup("username"))),
+      "-p", sensitive(Kamal::Utils.escape_shell_value(lookup("password")))
   end
 
   def logout

lib/kamal/commands/traefik.rb

@@ -1,11 +1,19 @@
 class Kamal::Commands::Traefik < Kamal::Commands::Base
-  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
-  DEFAULT_IMAGE = "traefik:v2.9"
+  DEFAULT_IMAGE = "traefik:v2.10"
   CONTAINER_PORT = 80
   DEFAULT_ARGS = {
     'log.level' => 'DEBUG'
   }
+  DEFAULT_LABELS = {
+    # These ensure we serve a 502 rather than a 404 if no containers are available
+    "traefik.http.routers.catchall.entryPoints" => "http",
+    "traefik.http.routers.catchall.rule" => "PathPrefix(`/`)",
+    "traefik.http.routers.catchall.service" => "unavailable",
+    "traefik.http.routers.catchall.priority" => 1,
+    "traefik.http.services.unavailable.loadbalancer.server.port" => "0"
+  }
 
   def run
     docker :run, "--name traefik",
@@ -31,7 +39,7 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
   end
 
   def start_or_run
-    combine start, run, by: "||"
+    any start, run
   end
 
   def info
@@ -64,7 +72,7 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
   end
 
   def env_file
-    env_file_with_secrets config.traefik.fetch("env", {})
+    Kamal::EnvFile.new(config.traefik.fetch("env", {}))
   end
 
   def host_env_file_path
@@ -97,7 +105,7 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
     end
 
     def labels
-      config.traefik["labels"] || []
+      DEFAULT_LABELS.merge(config.traefik["labels"] || {})
     end
 
     def image

lib/kamal/configuration.rb

@@ -6,11 +6,10 @@ require "erb"
 require "net/ssh/proxy/jump"
 
 class Kamal::Configuration
-  delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
+  delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, :logging, to: :raw_config, allow_nil: true
   delegate :argumentize, :optionize, to: Kamal::Utils
 
-  attr_accessor :destination
-  attr_accessor :raw_config
+  attr_reader :destination, :raw_config
 
   class << self
     def create_from(config_file:, destination: nil, version: nil)
@@ -26,7 +25,9 @@ class Kamal::Configuration
 
       def load_config_file(file)
         if file.exist?
-          YAML.load(ERB.new(IO.read(file)).result).symbolize_keys
+          # Newer Psych doesn't load aliases by default
+          load_method = YAML.respond_to?(:unsafe_load) ? :unsafe_load : :load
+          YAML.send(load_method, ERB.new(IO.read(file)).result).symbolize_keys
         else
           raise "Configuration file not found in #{file}"
         end
@@ -54,21 +55,20 @@ class Kamal::Configuration
   end
 
   def abbreviated_version
-    Kamal::Utils.abbreviate_version(version)
-  end
-
-  def run_directory
-    raw_config.run_directory || ".kamal"
-  end
-
-  def run_directory_as_docker_volume
-    if Pathname.new(run_directory).absolute?
-      run_directory
-    else
-      File.join "$(pwd)", run_directory
+    if version
+      # Don't abbreviate <sha>_uncommitted_<etc>
+      if version.include?("_")
+        version
+      else
+        version[0...7]
+      end
     end
   end
 
+  def minimum_version
+    raw_config.minimum_version
+  end
+
 
   def roles
     @roles ||= role_names.collect { |role_name| Role.new(role_name, config: self) }
@@ -91,19 +91,34 @@ class Kamal::Configuration
     roles.flat_map(&:hosts).uniq
   end
 
-  def primary_web_host
-    role(:web).primary_host
+  def primary_host
+    primary_role&.primary_host
+  end
+
+  def primary_role_name
+    raw_config.primary_role || "web"
+  end
+
+  def primary_role
+    role(primary_role_name)
+  end
+
+  def allow_empty_roles?
+    raw_config.allow_empty_roles
+  end
+
+  def traefik_roles
+    roles.select(&:running_traefik?)
+  end
+
+  def traefik_role_names
+    traefik_roles.flat_map(&:name)
   end
 
   def traefik_hosts
-    roles.select(&:running_traefik?).flat_map(&:hosts).uniq
+    traefik_roles.flat_map(&:hosts).uniq
   end
 
-  def boot
-    Kamal::Configuration::Boot.new(config: self)
-  end
-
-
   def repository
     [ raw_config.registry["server"], image ].compact.join("/")
   end
@@ -120,6 +135,14 @@ class Kamal::Configuration
     "#{service}-#{version}"
   end
 
+  def require_destination?
+    raw_config.require_destination
+  end
+
+  def retain_containers
+    raw_config.retain_containers || 5
+  end
+
 
   def volume_args
     if raw_config.volumes.present?
@@ -130,15 +153,27 @@ class Kamal::Configuration
   end
 
   def logging_args
-    if raw_config.logging.present?
-      optionize({ "log-driver" => raw_config.logging["driver"] }.compact) +
-        argumentize("--log-opt", raw_config.logging["options"])
+    if logging.present?
+      optionize({ "log-driver" => logging["driver"] }.compact) +
+        argumentize("--log-opt", logging["options"])
     else
       argumentize("--log-opt", { "max-size" => "10m" })
     end
   end
 
 
+  def boot
+    Kamal::Configuration::Boot.new(config: self)
+  end
+
+  def builder
+    Kamal::Configuration::Builder.new(config: self)
+  end
+
+  def traefik
+    raw_config.traefik || {}
+  end
+
   def ssh
     Kamal::Configuration::Ssh.new(config: self)
   end
@@ -149,27 +184,56 @@ class Kamal::Configuration
 
 
   def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }.merge(raw_config.healthcheck || {})
+    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }.merge(raw_config.healthcheck || {})
+  end
+
+  def healthcheck_service
+    [ "healthcheck", service, destination ].compact.join("-")
   end
 
   def readiness_delay
     raw_config.readiness_delay || 7
   end
 
-  def minimum_version
-    raw_config.minimum_version
+  def run_id
+    @run_id ||= SecureRandom.hex(16)
   end
 
+
+  def run_directory
+    raw_config.run_directory || ".kamal"
+  end
+
+  def run_directory_as_docker_volume
+    if Pathname.new(run_directory).absolute?
+      run_directory
+    else
+      File.join "$(pwd)", run_directory
+    end
+  end
+
+  def hooks_path
+    raw_config.hooks_path || ".kamal/hooks"
+  end
+
+  def host_env_directory
+    "#{run_directory}/env"
+  end
+
+  def asset_path
+    raw_config.asset_path
+  end
+
+
   def valid?
-    ensure_required_keys_present && ensure_valid_kamal_version
+    ensure_destination_if_required && ensure_required_keys_present && ensure_valid_kamal_version && ensure_retain_containers_valid && ensure_valid_service_name
   end
 
-
   def to_h
     {
       roles: role_names,
       hosts: all_hosts,
-      primary_host: primary_web_host,
+      primary_host: primary_host,
       version: version,
       repository: repository,
       absolute_image: absolute_image,
@@ -184,39 +248,17 @@ class Kamal::Configuration
     }.compact
   end
 
-  def traefik
-    raw_config.traefik || {}
-  end
-
-  def hooks_path
-    raw_config.hooks_path || ".kamal/hooks"
-  end
-
-  def builder
-    Kamal::Configuration::Builder.new(config: self)
-  end
-
-  # Will raise KeyError if any secret ENVs are missing
-  def ensure_env_available
-    roles.each(&:env_file)
-
-    true
-  end
-
-  def host_env_directory
-    "#{run_directory}/env"
-  end
-
-  def run_id
-    @run_id ||= SecureRandom.hex(16)
-  end
-
-  def asset_path
-    raw_config.asset_path
-  end
 
   private
     # Will raise ArgumentError if any required config keys are missing
+    def ensure_destination_if_required
+      if require_destination? && destination.nil?
+        raise ArgumentError, "You must specify a destination"
+      end
+
+      true
+    end
+
     def ensure_required_keys_present
       %i[ service image registry servers ].each do |key|
         raise ArgumentError, "Missing required configuration for #{key}" unless raw_config[key].present?
@@ -230,15 +272,31 @@ class Kamal::Configuration
         raise ArgumentError, "You must specify a password for the registry in config/deploy.yml (or set the ENV variable if that's used)"
       end
 
-      roles.each do |role|
-        if role.hosts.empty?
-          raise ArgumentError, "No servers specified for the #{role.name} role"
+      unless role_names.include?(primary_role_name)
+        raise ArgumentError, "The primary_role #{primary_role_name} isn't defined"
+      end
+
+      if primary_role.hosts.empty?
+        raise ArgumentError, "No servers specified for the #{primary_role.name} primary_role"
+      end
+
+      unless allow_empty_roles?
+        roles.each do |role|
+          if role.hosts.empty?
+            raise ArgumentError, "No servers specified for the #{role.name} role. You can ignore this with allow_empty_roles: true"
+          end
         end
       end
 
       true
     end
 
+    def ensure_valid_service_name
+      raise ArgumentError, "Service name can only include alphanumeric characters, hyphens, and underscores" unless raw_config[:service] =~ /^[a-z0-9_-]+$/
+
+      true
+    end
+
     def ensure_valid_kamal_version
       if minimum_version && Gem::Version.new(minimum_version) > Gem::Version.new(Kamal::VERSION)
         raise ArgumentError, "Current version is #{Kamal::VERSION}, minimum required is #{minimum_version}"
@@ -247,6 +305,12 @@ class Kamal::Configuration
       true
     end
 
+    def ensure_retain_containers_valid
+      raise ArgumentError, "Must retain at least 1 container" if retain_containers < 1
+
+      true
+    end
+
 
     def role_names
       raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort
@@ -254,10 +318,8 @@ class Kamal::Configuration
 
     def git_version
       @git_version ||=
-        if system("git rev-parse")
-          uncommitted_suffix = Kamal::Utils.uncommitted_changes.present? ? "_uncommitted_#{SecureRandom.hex(8)}" : ""
-
-          "#{`git rev-parse HEAD`.strip}#{uncommitted_suffix}"
+        if Kamal::Git.used?
+          [ Kamal::Git.revision, Kamal::Git.uncommitted_changes.present? ? "_uncommitted_#{SecureRandom.hex(8)}" : "" ].join
         else
           raise "Can't use commit hash as version, no git repository found in #{Dir.pwd}"
         end

lib/kamal/configuration/accessory.rb

@@ -1,5 +1,5 @@
 class Kamal::Configuration::Accessory
-  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_accessor :name, :specifics
 
@@ -8,7 +8,7 @@ class Kamal::Configuration::Accessory
   end
 
   def service_name
-    "#{config.service}-#{name}"
+    specifics["service"] || "#{config.service}-#{name}"
   end
 
   def image
@@ -46,7 +46,7 @@ class Kamal::Configuration::Accessory
   end
 
   def env_file
-    env_file_with_secrets env
+    Kamal::EnvFile.new(env)
   end
 
   def host_env_directory
@@ -70,8 +70,8 @@ class Kamal::Configuration::Accessory
 
   def directories
     specifics["directories"]&.to_h do |host_to_container_mapping|
-      host_relative_path, container_path = host_to_container_mapping.split(":")
-      [ expand_host_path(host_relative_path), container_path ]
+      host_path, container_path = host_to_container_mapping.split(":")
+      [ expand_host_path(host_path), container_path ]
     end || {}
   end
 
@@ -138,13 +138,17 @@ class Kamal::Configuration::Accessory
 
     def remote_directories_as_volumes
       specifics["directories"]&.collect do |host_to_container_mapping|
-        host_relative_path, container_path = host_to_container_mapping.split(":")
-        [ expand_host_path(host_relative_path), container_path ].join(":")
+        host_path, container_path = host_to_container_mapping.split(":")
+        [ expand_host_path(host_path), container_path ].join(":")
       end || []
     end
 
-    def expand_host_path(host_relative_path)
-      "#{service_data_directory}/#{host_relative_path}"
+    def expand_host_path(host_path)
+      absolute_path?(host_path) ? host_path : "#{service_data_directory}/#{host_path}"
+    end
+
+    def absolute_path?(path)
+      Pathname.new(path).absolute?
     end
 
     def service_data_directory

lib/kamal/configuration/boot.rb

@@ -8,7 +8,7 @@ class Kamal::Configuration::Boot
     limit = @options["limit"]
 
     if limit.to_s.end_with?("%")
-      @host_count * limit.to_i / 100
+      [@host_count * limit.to_i / 100, 1].max
     else
       limit
     end

lib/kamal/configuration/builder.rb

@@ -81,6 +81,10 @@ class Kamal::Configuration::Builder
     end
   end
 
+  def ssh
+    @options["ssh"]
+  end
+
   private
     def valid?
       if @options["cache"] && @options["cache"]["type"]

lib/kamal/configuration/role.rb

@@ -1,11 +1,12 @@
 class Kamal::Configuration::Role
   CORD_FILE = "cord"
-  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_accessor :name
+  alias to_s name
 
   def initialize(name, config:)
-   @name, @config = name.inquiry, config
+    @name, @config = name.inquiry, config
   end
 
   def primary_host
@@ -16,6 +17,18 @@ class Kamal::Configuration::Role
     @hosts ||= extract_hosts_from_config
   end
 
+  def cmd
+    specializations["cmd"]
+  end
+
+  def option_args
+    if args = specializations["options"]
+      optionize args
+    else
+      []
+    end
+  end
+
   def labels
     default_labels.merge(traefik_labels).merge(custom_labels)
   end
@@ -24,6 +37,19 @@ class Kamal::Configuration::Role
     argumentize "--label", labels
   end
 
+  def logging_args
+    args = config.logging || {}
+    args.deep_merge!(specializations["logging"]) if specializations["logging"].present?
+
+    if args.any?
+      optionize({ "log-driver" => args["driver"] }.compact) +
+        argumentize("--log-opt", args["options"])
+    else
+      config.logging_args
+    end
+  end
+
+
   def env
     if config.env && config.env["secret"]
       merged_env_with_secrets
@@ -33,7 +59,7 @@ class Kamal::Configuration::Role
   end
 
   def env_file
-    env_file_with_secrets env
+    Kamal::EnvFile.new(env)
   end
 
   def host_env_directory
@@ -52,6 +78,7 @@ class Kamal::Configuration::Role
     asset_volume&.docker_args
   end
 
+
   def health_check_args(cord: true)
     if health_check_cmd.present?
       if cord && uses_cord?
@@ -77,6 +104,20 @@ class Kamal::Configuration::Role
     health_check_options["interval"] || "1s"
   end
 
+
+  def running_traefik?
+    if specializations["traefik"].nil?
+      primary?
+    else
+      specializations["traefik"]
+    end
+  end
+
+  def primary?
+    self == @config.primary_role
+  end
+
+
   def uses_cord?
     running_traefik? && cord_volume && health_check_cmd.present?
   end
@@ -106,22 +147,6 @@ class Kamal::Configuration::Role
   end
 
 
-  def cmd
-    specializations["cmd"]
-  end
-
-  def option_args
-    if args = specializations["options"]
-      optionize args
-    else
-      []
-    end
-  end
-
-  def running_traefik?
-    name.web? || specializations["traefik"]
-  end
-
   def container_name(version = nil)
     [ container_prefix, version || config.version ].compact.join("-")
   end
@@ -130,6 +155,7 @@ class Kamal::Configuration::Role
     [ config.service, name, config.destination ].compact.join("-")
   end
 
+
   def asset_path
     specializations["asset_path"] || config.asset_path
   end
@@ -180,6 +206,7 @@ class Kamal::Configuration::Role
           "traefik.http.services.#{traefik_service}.loadbalancer.server.scheme" => "http",
 
           "traefik.http.routers.#{traefik_service}.rule" => "PathPrefix(`/`)",
+          "traefik.http.routers.#{traefik_service}.priority" => "2",
           "traefik.http.middlewares.#{traefik_service}-retry.retry.attempts" => "5",
           "traefik.http.middlewares.#{traefik_service}-retry.retry.initialinterval" => "500ms",
           "traefik.http.routers.#{traefik_service}.middlewares" => "#{traefik_service}-retry@docker"
@@ -225,7 +252,7 @@ class Kamal::Configuration::Role
         clear_app_env  = config.env["secret"] ? Array(config.env["clear"]) : Array(config.env["clear"] || config.env)
         clear_role_env = specialized_env["secret"] ? Array(specialized_env["clear"]) : Array(specialized_env["clear"] || specialized_env)
 
-        new_env["clear"] = (clear_app_env + clear_role_env).uniq
+        new_env["clear"] = clear_app_env.to_h.merge(clear_role_env.to_h)
       end
     end
 

lib/kamal/configuration/ssh.rb

@@ -9,6 +9,10 @@ class Kamal::Configuration::Ssh
     config.fetch("user", "root")
   end
 
+  def port
+    config.fetch("port", 22)
+  end
+
   def proxy
     if (proxy = config["proxy"])
       Net::SSH::Proxy::Jump.new(proxy.include?("@") ? proxy : "root@#{proxy}")
@@ -18,7 +22,7 @@ class Kamal::Configuration::Ssh
   end
 
   def options
-    { user: user, proxy: proxy, auth_methods: [ "publickey" ], logger: logger, keepalive: true, keepalive_interval: 30 }.compact
+    { user: user, port: port, proxy: proxy, logger: logger, keepalive: true, keepalive_interval: 30 }.compact
   end
 
   def to_h

lib/kamal/env_file.rb

@@ -0,0 +1,41 @@
+# Encode an env hash as a string where secret values have been looked up and all values escaped for Docker.
+class Kamal::EnvFile
+  def initialize(env)
+    @env = env
+  end
+  
+  def to_s
+    env_file = StringIO.new.tap do |contents|
+      if (secrets = @env["secret"]).present?
+        @env.fetch("secret", @env)&.each do |key|
+          contents << docker_env_file_line(key, ENV.fetch(key))
+        end
+
+        @env["clear"]&.each do |key, value|
+          contents << docker_env_file_line(key, value)
+        end
+      else
+        @env.fetch("clear", @env)&.each do |key, value|
+          contents << docker_env_file_line(key, value)
+        end
+      end
+    end.string
+
+    # Ensure the file has some contents to avoid the SSHKIT empty file warning
+    env_file.presence || "\n"
+  end
+
+  alias to_str to_s
+  
+  private
+    def docker_env_file_line(key, value)
+      "#{key.to_s}=#{escape_docker_env_file_value(value)}\n"
+    end
+
+    # Escape a value to make it safe to dump in a docker file.
+    def escape_docker_env_file_value(value)
+      # Doublequotes are treated literally in docker env files
+      # so remove leading and trailing ones and unescape any others
+      value.to_s.dump[1..-2].gsub(/\\"/, "\"")
+    end
+end

lib/kamal/git.rb

@@ -0,0 +1,19 @@
+module Kamal::Git
+  extend self
+
+  def used?
+    system("git rev-parse")
+  end
+
+  def user_name
+    `git config user.name`.strip
+  end
+
+  def revision
+    `git rev-parse HEAD`.strip
+  end
+
+  def uncommitted_changes
+    `git status --porcelain`.strip
+  end
+end

lib/kamal/sshkit_with_ext.rb

@@ -1,5 +1,6 @@
 require "sshkit"
 require "sshkit/dsl"
+require "net/scp"
 require "active_support/core_ext/hash/deep_merge"
 require "json"
 

lib/kamal/utils.rb

@@ -16,26 +16,6 @@ module Kamal::Utils
     end
   end
 
-  def env_file_with_secrets(env)
-    env_file = StringIO.new.tap do |contents|
-      if (secrets = env["secret"]).present?
-        env.fetch("secret", env)&.each do |key|
-          contents << docker_env_file_line(key, ENV.fetch(key))
-        end
-        env["clear"]&.each do |key, value|
-          contents << docker_env_file_line(key, value)
-        end
-      else
-        env.fetch("clear", env)&.each do |key, value|
-          contents << docker_env_file_line(key, value)
-        end
-      end
-    end.string
-
-    # Ensure the file has some contents to avoid the SSHKIT empty file warning
-    env_file || "\n"
-  end
-
   # Returns a list of shell-dashed option arguments. If the value is true, it's treated like a value-less option.
   def optionize(args, with: nil)
     options = if with
@@ -72,19 +52,6 @@ module Kamal::Utils
     end
   end
 
-  def unredacted(value)
-    case
-    when value.respond_to?(:unredacted)
-      value.unredacted
-    when value.respond_to?(:transform_values)
-      value.transform_values { |value| unredacted value }
-    when value.respond_to?(:map)
-      value.map { |element| unredacted element }
-    else
-      value
-    end
-  end
-
   # Escape a value to make it safe for shell use.
   def escape_shell_value(value)
     value.to_s.dump
@@ -92,27 +59,19 @@ module Kamal::Utils
       .gsub(DOLLAR_SIGN_WITHOUT_SHELL_EXPANSION_REGEX, '\$')
   end
 
-  # Abbreviate a git revhash for concise display
-  def abbreviate_version(version)
-    if version
-      # Don't abbreviate <sha>_uncommitted_<etc>
-      if version.include?("_")
-        version
-      else
-        version[0...7]
+  # Apply a list of host or role filters, including wildcard matches
+  def filter_specific_items(filters, items)
+    matches = []
+
+    Array(filters).select do |filter|
+      matches += Array(items).select do |item|
+        # Only allow * for a wildcard
+        pattern = Regexp.escape(filter).gsub('\*', '.*')
+        # items are roles or hosts
+        (item.respond_to?(:name) ? item.name : item).match(/^#{pattern}$/)
       end
     end
-  end
 
-  def uncommitted_changes
-    `git status --porcelain`.strip
-  end
-
-  def docker_env_file_line(key, value)
-    if key.include?("\n") || value.to_s.include?("\n")
-      raise ArgumentError, "docker env file format does not support newlines in keys or values, key: #{key}"
-    end
-
-    "#{key.to_s}=#{value.to_s}\n"
+    matches
   end
 end

lib/kamal/utils/healthcheck_poller.rb

@@ -1,64 +0,0 @@
-class Kamal::Utils::HealthcheckPoller
-  TRAEFIK_UPDATE_DELAY = 2
-
-  class HealthcheckError < StandardError; end
-
-  class << self
-    def wait_for_healthy(pause_after_ready: false, &block)
-      attempt = 1
-      max_attempts = KAMAL.config.healthcheck["max_attempts"]
-
-      begin
-        case status = block.call
-        when "healthy"
-          sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
-        when "running" # No health check configured
-          sleep KAMAL.config.readiness_delay if pause_after_ready
-        else
-          raise HealthcheckError, "container not ready (#{status})"
-        end
-      rescue HealthcheckError => e
-        if attempt <= max_attempts
-          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-          sleep attempt
-          attempt += 1
-          retry
-        else
-          raise
-        end
-      end
-
-      info "Container is healthy!"
-    end
-
-    def wait_for_unhealthy(pause_after_ready: false, &block)
-      attempt = 1
-      max_attempts = KAMAL.config.healthcheck["max_attempts"]
-
-      begin
-        case status = block.call
-        when "unhealthy"
-          sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
-        else
-          raise HealthcheckError, "container not unhealthy (#{status})"
-        end
-      rescue HealthcheckError => e
-        if attempt <= max_attempts
-          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-          sleep attempt
-          attempt += 1
-          retry
-        else
-          raise
-        end
-      end
-
-      info "Container is unhealthy!"
-    end
-
-    private
-      def info(message)
-        SSHKit.config.output.info(message)
-      end
-  end
-end

lib/kamal/utils/sensitive.rb

@@ -1,4 +1,5 @@
 require "active_support/core_ext/module/delegation"
+require "sshkit"
 
 class Kamal::Utils::Sensitive
   # So SSHKit knows to redact these values.

lib/kamal/version.rb

@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "0.16.1"
+  VERSION = "1.4.0"
 end

test/cli/accessory_test.rb

@@ -48,6 +48,18 @@ class CliAccessoryTest < CliTestCase
     run_command("reboot", "mysql")
   end
 
+  test "reboot all" do
+    Kamal::Commands::Registry.any_instance.expects(:login).times(3)
+    Kamal::Cli::Accessory.any_instance.expects(:stop).with("mysql")
+    Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("mysql")
+    Kamal::Cli::Accessory.any_instance.expects(:boot).with("mysql", login: false)
+    Kamal::Cli::Accessory.any_instance.expects(:stop).with("redis")
+    Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("redis")
+    Kamal::Cli::Accessory.any_instance.expects(:boot).with("redis", login: false)
+
+    run_command("reboot", "all")
+  end
+
   test "start" do
     assert_match "docker container start app-mysql", run_command("start", "mysql")
   end
@@ -97,7 +109,7 @@ class CliAccessoryTest < CliTestCase
 
   test "logs with follow" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.3 'docker logs app-mysql --timestamps --tail 10 --follow 2>&1'")
+      .with("ssh -t root@1.1.1.3 -p 22 'docker logs app-mysql --timestamps --tail 10 --follow 2>&1'")
 
     assert_match "docker logs app-mysql --timestamps --tail 10 --follow 2>&1", run_command("logs", "mysql", "--follow")
   end
@@ -136,6 +148,30 @@ class CliAccessoryTest < CliTestCase
     assert_match "rm -rf app-mysql", run_command("remove_service_directory", "mysql")
   end
 
+  test "hosts param respected" do
+    Kamal::Cli::Accessory.any_instance.expects(:directories).with("redis")
+    Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis")
+
+    run_command("boot", "redis", "--hosts", "1.1.1.1").tap do |output|
+      assert_match /docker login.*on 1.1.1.1/, output
+      refute_match /docker login.*on 1.1.1.2/, output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
+      refute_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
+    end
+  end
+
+  test "hosts param intersected with configuration" do
+    Kamal::Cli::Accessory.any_instance.expects(:directories).with("redis")
+    Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis")
+
+    run_command("boot", "redis", "--hosts", "1.1.1.1,1.1.1.3").tap do |output|
+      assert_match /docker login.*on 1.1.1.1/, output
+      refute_match /docker login.*on 1.1.1.3/, output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
+      refute_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.3", output
+    end
+  end
+
   private
     def run_command(*command)
       stdouted { Kamal::Cli::Accessory.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }

test/cli/app_test.rb

@@ -27,7 +27,7 @@ class CliAppTest < CliTestCase
       .returns("123") # old version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
       .returns("cordfile") # old version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -55,8 +55,6 @@ class CliAppTest < CliTestCase
   end
 
   test "boot errors leave lock in place" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
-
     Kamal::Cli::App.any_instance.expects(:using_version).raises(RuntimeError)
 
     assert !KAMAL.holding_lock?
@@ -66,6 +64,34 @@ class CliAppTest < CliTestCase
     assert KAMAL.holding_lock?
   end
 
+  test "boot with assets" do
+    Object.any_instance.stubs(:sleep)
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678") # running version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("123").twice # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .returns("") # old version
+
+    run_command("boot", config: :with_assets).tap do |output|
+      assert_match "docker tag dhh/app:latest dhh/app:latest", output
+      assert_match "/usr/bin/env mkdir -p .kamal/assets/volumes/app-web-latest ; cp -rnT .kamal/assets/extracted/app-web-latest .kamal/assets/volumes/app-web-latest ; cp -rnT .kamal/assets/extracted/app-web-latest .kamal/assets/volumes/app-web-123 || true ; cp -rnT .kamal/assets/extracted/app-web-123 .kamal/assets/volumes/app-web-latest || true", output
+      assert_match "/usr/bin/env mkdir -p .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets 2> /dev/null || true && docker run --name app-web-assets --detach --rm dhh/app:latest sleep 1000000 && docker cp -L app-web-assets:/public/assets/. .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets", output
+      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} /, output
+      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+      assert_match "/usr/bin/env find .kamal/assets/extracted -maxdepth 1 -name 'app-web-*' ! -name app-web-latest -exec rm -rf \"{}\" + ; find .kamal/assets/volumes -maxdepth 1 -name 'app-web-*' ! -name app-web-latest -exec rm -rf \"{}\" +", output
+    end
+  end
+
   test "start" do
     run_command("start").tap do |output|
       assert_match "docker start app-web-999", output
@@ -133,7 +159,7 @@ class CliAppTest < CliTestCase
 
   test "exec" do
     run_command("exec", "ruby -v").tap do |output|
-      assert_match "docker run --rm dhh/app:latest ruby -v", output
+      assert_match "docker run --rm --env-file .kamal/env/roles/app-web.env dhh/app:latest ruby -v", output
     end
   end
 
@@ -144,6 +170,25 @@ class CliAppTest < CliTestCase
     end
   end
 
+  test "exec interactive" do
+    SSHKit::Backend::Abstract.any_instance.expects(:exec)
+      .with("ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --env-file .kamal/env/roles/app-web.env dhh/app:latest ruby -v'")
+    run_command("exec", "-i", "ruby -v").tap do |output|
+      assert_match "Get most recent version available as an image...", output
+      assert_match "Launching interactive command with version latest via SSH from new container on 1.1.1.1...", output
+    end
+  end
+
+  test "exec interactive with reuse" do
+    SSHKit::Backend::Abstract.any_instance.expects(:exec)
+      .with("ssh -t root@1.1.1.1 -p 22 'docker exec -it app-web-999 ruby -v'")
+    run_command("exec", "-i", "--reuse", "ruby -v").tap do |output|
+      assert_match "Get current version of running container...", output
+      assert_match "Running docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done on 1.1.1.1", output
+      assert_match "Launching interactive command with version 999 via SSH from existing container on 1.1.1.1...", output
+    end
+  end
+
   test "containers" do
     run_command("containers").tap do |output|
       assert_match "docker container ls --all --filter label=service=app", output
@@ -165,7 +210,7 @@ class CliAppTest < CliTestCase
 
   test "logs with follow" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
+      .with("ssh -t root@1.1.1.1 -p 22 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
 
     assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
   end

test/cli/build_test.rb

@@ -57,6 +57,7 @@ class CliBuildTest < CliTestCase
     run_command("pull").tap do |output|
       assert_match /docker image rm --force dhh\/app:999/, output
       assert_match /docker pull dhh\/app:999/, output
+      assert_match "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:999 | grep -x app || (echo \"Image dhh/app:999 is missing the 'service' label\" && exit 1)", output
     end
   end
 
@@ -66,6 +67,14 @@ class CliBuildTest < CliTestCase
     end
   end
 
+  test "create remote" do
+    run_command("create", fixture: :with_remote_builder).tap do |output|
+      assert_match "Running /usr/bin/env true on 1.1.1.5", output
+      assert_match "docker context create kamal-app-native-remote-amd64 --description 'kamal-app-native-remote amd64 native host' --docker 'host=ssh://app@1.1.1.5'", output
+      assert_match "docker buildx create --name kamal-app-native-remote kamal-app-native-remote-amd64 --platform linux/amd64", output
+    end
+  end
+
   test "create with error" do
     stub_setup
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
@@ -95,8 +104,8 @@ class CliBuildTest < CliTestCase
   end
 
   private
-    def run_command(*command)
-      stdouted { Kamal::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+    def run_command(*command, fixture: :with_accessories)
+      stdouted { Kamal::Cli::Build.start([*command, "-c", "test/fixtures/deploy_#{fixture}.yml"]) }
     end
 
     def stub_dependency_checks

test/cli/healthcheck_test.rb

@@ -5,7 +5,7 @@ class CliHealthcheckTest < CliTestCase
     # Prevent expected failures from outputting to terminal
     Thread.report_on_exception = false
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
     Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
@@ -35,7 +35,7 @@ class CliHealthcheckTest < CliTestCase
     # Prevent expected failures from outputting to terminal
     Thread.report_on_exception = false
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
@@ -64,9 +64,19 @@ class CliHealthcheckTest < CliTestCase
     end
     assert_match "container not ready (unhealthy)", exception.message
   end
+  
+  test "raises an exception if primary does not have traefik" do
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).never
+    
+    exception = assert_raises do
+      run_command("perform", config_file: "test/fixtures/deploy_workers_only.yml")
+    end
+
+    assert_equal "The primary host is not configured to run Traefik", exception.message
+  end
 
   private
-    def run_command(*command)
-      stdouted { Kamal::Cli::Healthcheck.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+    def run_command(*command, config_file: "test/fixtures/deploy_with_accessories.yml")
+      stdouted { Kamal::Cli::Healthcheck.start([*command, "-c", config_file]) } 
     end
 end

test/cli/main_test.rb

@@ -2,11 +2,47 @@ require_relative "cli_test_case"
 
 class CliMainTest < CliTestCase
   test "setup" do
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap")
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ])
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
+    
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:deploy)
 
-    run_command("setup")
+    run_command("setup").tap do |output|
+      assert_match /Ensure Docker is installed.../, output
+      assert_match /Push env files.../, output
+    end
+  end
+
+  test "setup with skip_push" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
+    # deploy
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
+
+    run_command("setup", "--skip_push").tap do |output|
+      assert_match /Ensure Docker is installed.../, output
+      assert_match /Push env files.../, output
+      # deploy
+      assert_match /Acquiring the deploy lock/, output
+      assert_match /Log into image registry/, output
+      assert_match /Pull app image/, output
+      assert_match /Ensure Traefik is running/, output
+      assert_match /Ensure app can pass healthcheck/, output
+      assert_match /Detect stale containers/, output
+      assert_match /Prune old containers and images/, output
+      assert_match /Releasing the deploy lock/, output
+    end
   end
 
   test "deploy" do
@@ -16,7 +52,7 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
@@ -43,7 +79,7 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
@@ -113,7 +149,7 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
@@ -121,11 +157,34 @@ class CliMainTest < CliTestCase
       refute_match /Running the post-deploy hook.../, output
     end
   end
+  
+  test "deploy without healthcheck if primary host doesn't have traefik" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_workers_only.yml", "version" => "999", "skip_hooks" => false }
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options).never
+    
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
+
+    run_command("deploy", config_file: "deploy_workers_only")
+  end
 
   test "deploy with missing secrets" do
-    assert_raises(KeyError) do
-      run_command("deploy", config_file: "deploy_with_secrets")
-    end
+    invoke_options = { "config_file" => "test/fixtures/deploy_with_secrets.yml", "version" => "999", "skip_hooks" => false }
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
+
+    run_command("deploy", config_file: "deploy_with_secrets")
   end
 
   test "redeploy" do
@@ -133,7 +192,7 @@ class CliMainTest < CliTestCase
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
 
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
@@ -155,7 +214,7 @@ class CliMainTest < CliTestCase
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
 
     run_command("redeploy", "--skip_push").tap do |output|
@@ -193,7 +252,7 @@ class CliMainTest < CliTestCase
     end
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
       .returns("corddirectory").at_least_once # health check
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -215,7 +274,7 @@ class CliMainTest < CliTestCase
   test "rollback without old version" do
     Kamal::Cli::Main.any_instance.stubs(:container_available?).returns(true)
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep)
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep)
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
@@ -274,6 +333,16 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "config with primary web role override" do
+    run_command("config", config_file: "deploy_primary_web_role_override").tap do |output|
+      config = YAML.load(output)
+
+      assert_equal ["web_chicago", "web_tokyo"], config[:roles]
+      assert_equal ["1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4"], config[:hosts]
+      assert_equal "1.1.1.3", config[:primary_host]
+    end
+  end
+
   test "config with destination" do
     run_command("config", "-d", "world", config_file: "deploy_for_dest").tap do |output|
       config = YAML.load(output)
@@ -287,6 +356,19 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "config with aliases" do
+    run_command("config", config_file: "deploy_with_aliases").tap do |output|
+      config = YAML.load(output)
+
+      assert_equal ["web", "web_tokyo", "workers", "workers_tokyo"], config[:roles]
+      assert_equal ["1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4"], config[:hosts]
+      assert_equal "999", config[:version]
+      assert_equal "registry.digitalocean.com/dhh/app", config[:repository]
+      assert_equal "registry.digitalocean.com/dhh/app:999", config[:absolute_image]
+      assert_equal "app-999", config[:service_with_version]
+    end
+  end
+
   test "init" do
     Pathname.any_instance.expects(:exist?).returns(false).times(3)
     Pathname.any_instance.stubs(:mkpath)
@@ -345,6 +427,20 @@ class CliMainTest < CliTestCase
     run_command("envify")
   end
 
+  test "envify with blank line trimming" do
+    file = <<~EOF
+      HELLO=<%= 'world' %>
+      <% if true -%>
+      KEY=value
+      <% end  -%>
+    EOF
+
+    File.expects(:read).with(".env.erb").returns(file.strip)
+    File.expects(:write).with(".env", "HELLO=world\nKEY=value\n", perm: 0600)
+
+    run_command("envify")
+  end
+
   test "envify with destination" do
     File.expects(:read).with(".env.world.erb").returns("HELLO=<%= 'world' %>")
     File.expects(:write).with(".env.world", "HELLO=world", perm: 0600)
@@ -352,6 +448,14 @@ class CliMainTest < CliTestCase
     run_command("envify", "-d", "world", config_file: "deploy_for_dest")
   end
 
+  test "envify with skip_push" do
+    File.expects(:read).with(".env.erb").returns("HELLO=<%= 'world' %>")
+    File.expects(:write).with(".env", "HELLO=world", perm: 0600)
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push").never
+    run_command("envify", "--skip-push")
+  end
+
   test "remove with confirmation" do
     run_command("remove", "-y", config_file: "deploy_with_accessories").tap do |output|
       assert_match /docker container stop traefik/, output

test/cli/prune_test.rb

@@ -10,7 +10,7 @@ class CliPruneTest < CliTestCase
 
   test "images" do
     run_command("images").tap do |output|
-      assert_match "docker image prune --force --filter label=service=app --filter dangling=true on 1.1.1.", output
+      assert_match "docker image prune --force --filter label=service=app on 1.1.1.", output
       assert_match "docker image ls --filter label=service=app --format '{{.ID}} {{.Repository}}:{{.Tag}}' | grep -v -w \"$(docker container ls -a --format '{{.Image}}\\|' --filter label=service=app | tr -d '\\n')dhh/app:latest\\|dhh/app:<none>\" | while read image tag; do docker rmi $tag; done on 1.1.1.", output
     end
   end
@@ -18,6 +18,16 @@ class CliPruneTest < CliTestCase
   test "containers" do
     run_command("containers").tap do |output|
       assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output
+      assert_match /docker container prune --force --filter label=service=healthcheck-app on 1.1.1.\d/, output
+     end
+
+    run_command("containers", "--retain", "10").tap do |output|
+      assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +11 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output
+      assert_match /docker container prune --force --filter label=service=healthcheck-app on 1.1.1.\d/, output
+    end
+
+    assert_raises(RuntimeError, "retain must be at least 1") do
+      run_command("containers", "--retain", "0")
     end
   end
 

test/cli/server_test.rb

@@ -10,7 +10,7 @@ class CliServerTest < CliTestCase
 
   test "bootstrap install as non-root user" do
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ]', raise_on_non_zero_exit: false).returns(false).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', raise_on_non_zero_exit: false).returns(false).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
 
     assert_raise RuntimeError, "Docker is not installed on 1.1.1.1, 1.1.1.3, 1.1.1.4, 1.1.1.2 and can't be automatically installed without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/" do
@@ -20,13 +20,16 @@ class CliServerTest < CliTestCase
 
   test "bootstrap install as root user" do
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ]', raise_on_non_zero_exit: false).returns(true).at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:curl, "-fsSL", "https://get.docker.com", "|", :sh).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', raise_on_non_zero_exit: false).returns(true).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:sh, "-c", "'curl -fsSL https://get.docker.com || wget -O - https://get.docker.com || echo \"exit 1\"'", "|", :sh).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
+    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(".kamal/hooks/docker-setup", anything).at_least_once
 
     run_command("bootstrap").tap do |output|
       ("1.1.1.1".."1.1.1.4").map do |host|
         assert_match "Missing Docker on #{host}. Installing…", output
+        assert_match "Running the docker-setup hook", output
       end
     end
   end

test/cli/traefik_test.rb

@@ -4,7 +4,7 @@ class CliTraefikTest < CliTestCase
   test "boot" do
     run_command("boot").tap do |output|
       assert_match "docker login", output
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
+      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
     end
   end
 
@@ -14,7 +14,7 @@ class CliTraefikTest < CliTestCase
     run_command("reboot").tap do |output|
       assert_match "docker container stop traefik", output
       assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik", output
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
+      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
     end
   end
 
@@ -64,7 +64,7 @@ class CliTraefikTest < CliTestCase
 
   test "logs with follow" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 'docker logs traefik --timestamps --tail 10 --follow 2>&1'")
+      .with("ssh -t root@1.1.1.1 -p 22 'docker logs traefik --timestamps --tail 10 --follow 2>&1'")
 
     assert_match "docker logs traefik --timestamps --tail 10 --follow", run_command("logs", "--follow")
   end

test/commander_test.rb

@@ -14,6 +14,20 @@ class CommanderTest < ActiveSupport::TestCase
 
     @kamal.specific_hosts = [ "1.1.1.1", "1.1.1.2" ]
     assert_equal [ "1.1.1.1", "1.1.1.2" ], @kamal.hosts
+
+    @kamal.specific_hosts = [ "1.1.1.1*" ]
+    assert_equal [ "1.1.1.1" ], @kamal.hosts
+
+    @kamal.specific_hosts = [ "1.1.1.*", "*.1.2.*" ]
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @kamal.hosts
+
+    @kamal.specific_hosts = [ "*" ]
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @kamal.hosts
+
+    exception = assert_raises(ArgumentError) do
+      @kamal.specific_hosts = [ "*miss" ]
+    end
+    assert_match /hosts match for \*miss/, exception.message
   end
 
   test "filtering hosts by filtering roles" do
@@ -21,6 +35,11 @@ class CommanderTest < ActiveSupport::TestCase
 
     @kamal.specific_roles = [ "web" ]
     assert_equal [ "1.1.1.1", "1.1.1.2" ], @kamal.hosts
+
+    exception = assert_raises(ArgumentError) do
+      @kamal.specific_roles = [ "*miss" ]
+    end
+    assert_match /roles match for \*miss/, exception.message
   end
 
   test "filtering roles" do
@@ -28,6 +47,20 @@ class CommanderTest < ActiveSupport::TestCase
 
     @kamal.specific_roles = [ "workers" ]
     assert_equal [ "workers" ], @kamal.roles.map(&:name)
+
+    @kamal.specific_roles = [ "w*" ]
+    assert_equal [ "web", "workers" ], @kamal.roles.map(&:name)
+
+    @kamal.specific_roles = [ "we*", "*orkers" ]
+    assert_equal [ "web", "workers" ], @kamal.roles.map(&:name)
+
+    @kamal.specific_roles = [ "*" ]
+    assert_equal [ "web", "workers" ], @kamal.roles.map(&:name)
+
+    exception = assert_raises(ArgumentError) do
+      @kamal.specific_roles = [ "*miss" ]
+    end
+    assert_match /roles match for \*miss/, exception.message
   end
 
   test "filtering roles by filtering hosts" do
@@ -49,9 +82,15 @@ class CommanderTest < ActiveSupport::TestCase
     assert_equal "1.1.1.3", @kamal.primary_host
   end
 
+  test "primary_role" do
+    assert_equal "web", @kamal.primary_role.name
+    @kamal.specific_roles = "workers"
+    assert_equal "workers", @kamal.primary_role.name
+  end
+
   test "roles_on" do
-    assert_equal [ "web" ], @kamal.roles_on("1.1.1.1")
-    assert_equal [ "workers" ], @kamal.roles_on("1.1.1.3")
+    assert_equal [ "web" ], @kamal.roles_on("1.1.1.1").map(&:name)
+    assert_equal [ "workers" ], @kamal.roles_on("1.1.1.3").map(&:name)
   end
 
   test "default group strategy" do
@@ -70,6 +109,21 @@ class CommanderTest < ActiveSupport::TestCase
     assert_equal({ in: :groups, limit: 1, wait: 2 }, @kamal.boot_strategy)
   end
 
+  test "percentage-based group strategy limit is at least 1" do
+    configure_with(:deploy_with_low_percentage_boot_strategy)
+
+    assert_equal({ in: :groups, limit: 1, wait: 2 }, @kamal.boot_strategy)
+  end
+
+  test "try to match the primary role from a list of specific roles" do
+    configure_with(:deploy_primary_web_role_override)
+
+    @kamal.specific_roles = [ "web_*" ]
+    assert_equal [ "web_chicago", "web_tokyo" ], @kamal.roles.map(&:name)
+    assert_equal "web_tokyo", @kamal.primary_role.name
+    assert_equal "1.1.1.3", @kamal.primary_host
+  end
+
   private
     def configure_with(variant)
       @kamal = Kamal::Commander.new.tap do |kamal|

test/commands/accessory_test.rb

@@ -34,6 +34,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
           ]
         },
         "busybox" => {
+          "service" => "custom-busybox",
           "image" => "busybox:latest",
           "host" => "1.1.1.7"
         }
@@ -57,7 +58,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
       new_command(:redis).run.join(" ")
 
     assert_equal \
-      "docker run --name app-busybox --detach --restart unless-stopped --log-opt max-size=\"10m\" --env-file .kamal/env/accessories/app-busybox.env --label service=\"app-busybox\" busybox:latest",
+      "docker run --name custom-busybox --detach --restart unless-stopped --log-opt max-size=\"10m\" --env-file .kamal/env/accessories/custom-busybox.env --label service=\"custom-busybox\" busybox:latest",
       new_command(:busybox).run.join(" ")
   end
 
@@ -65,7 +66,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --name app-busybox --detach --restart unless-stopped --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --env-file .kamal/env/accessories/app-busybox.env --label service=\"app-busybox\" busybox:latest",
+      "docker run --name custom-busybox --detach --restart unless-stopped --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --env-file .kamal/env/accessories/custom-busybox.env --label service=\"custom-busybox\" busybox:latest",
       new_command(:busybox).run.join(" ")
   end
 
@@ -128,7 +129,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "follow logs" do
     assert_equal \
-      "ssh -t root@1.1.1.5 'docker logs app-mysql --timestamps --tail 10 --follow 2>&1'",
+      "ssh -t root@1.1.1.5 -p 22 'docker logs app-mysql --timestamps --tail 10 --follow 2>&1'",
       new_command(:mysql).follow_logs
   end
 

test/commands/app_test.rb

@@ -14,13 +14,13 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with hostname" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run(hostname: "myhost").join(" ")
   end
 
@@ -28,7 +28,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:volumes] = ["/local/path:/container/path" ]
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -36,7 +36,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "path" => "/healthz" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -44,7 +44,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "cmd" => "/bin/up" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -52,14 +52,14 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with custom options" do
     @config[:servers] = { "web" => [ "1.1.1.1" ], "jobs" => { "hosts" => [ "1.1.1.2" ], "cmd" => "bin/jobs", "options" => { "mount" => "somewhere", "cap-add" => true } } }
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" --env-file .kamal/env/roles/app-jobs.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
+      "docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-jobs.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
       new_command(role: "jobs").run.join(" ")
   end
 
@@ -67,7 +67,16 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      new_command.run.join(" ")
+  end
+
+  test "run with role logging config" do
+    @config[:logging] = { "driver" => "local", "options" => { "max-size" => "10m", "max-file" => "3" } }
+    @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "logging" => { "driver" => "local", "options" => { "max-size" => "100m" } } } }
+
+    assert_equal \
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -145,12 +154,20 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "follow logs" do
     assert_match \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --follow 2>&1",
       new_command.follow_logs(host: "app-1")
 
     assert_match \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1 | grep \"Completed\"",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --follow 2>&1 | grep \"Completed\"",
       new_command.follow_logs(host: "app-1", grep: "Completed")
+
+    assert_match \
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 123 --follow 2>&1",
+      new_command.follow_logs(host: "app-1", lines: 123)
+
+    assert_match \
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 123 --follow 2>&1 | grep \"Completed\"",
+      new_command.follow_logs(host: "app-1", lines: 123, grep: "Completed")
   end
 
 
@@ -190,32 +207,37 @@ class CommandsAppTest < ActiveSupport::TestCase
   end
 
   test "run over ssh" do
-    assert_equal "ssh -t root@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with custom user" do
     @config[:ssh] = { "user" => "app" }
-    assert_equal "ssh -t app@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -t app@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+  end
+
+  test "run over ssh with custom port" do
+    @config[:ssh] = { "port" => "2222" }
+    assert_equal "ssh -t root@1.1.1.1 -p 2222 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with proxy" do
     @config[:ssh] = { "proxy" => "2.2.2.2" }
-    assert_equal "ssh -J root@2.2.2.2 -t root@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -J root@2.2.2.2 -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with proxy user" do
     @config[:ssh] = { "proxy" => "app@2.2.2.2" }
-    assert_equal "ssh -J app@2.2.2.2 -t root@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -J app@2.2.2.2 -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with custom user with proxy" do
     @config[:ssh] = { "user" => "app", "proxy" => "2.2.2.2" }
-    assert_equal "ssh -J root@2.2.2.2 -t app@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -J root@2.2.2.2 -t app@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with proxy_command" do
     @config[:ssh] = { "proxy_command" => "ssh -W %h:%p user@proxy-server" }
-    assert_equal "ssh -o ProxyCommand='ssh -W %h:%p user@proxy-server' -t root@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -o ProxyCommand='ssh -W %h:%p user@proxy-server' -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "current_running_container_id" do
@@ -317,10 +339,10 @@ class CommandsAppTest < ActiveSupport::TestCase
       new_command.remove_images.join(" ")
   end
 
-  test "tag_current_as_latest" do
+  test "tag_current_image_as_latest" do
     assert_equal \
       "docker tag dhh/app:999 dhh/app:latest",
-      new_command.tag_current_as_latest.join(" ")
+      new_command.tag_current_image_as_latest.join(" ")
   end
 
   test "make_env_directory" do
@@ -332,7 +354,7 @@ class CommandsAppTest < ActiveSupport::TestCase
   end
 
   test "cord" do
-    assert_equal "docker inspect -f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}' app-web-123 | awk '$2 == \"/tmp/kamal-cord\" {print $1}'", new_command.cord(version: 123).join(" ")
+    assert_equal "docker inspect -f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}' app-web-123 | awk '$2 == \"/tmp/kamal-cord\" {print $1}'", new_command.cord(version: 123).join(" ")
   end
 
   test "tie cord" do
@@ -345,8 +367,40 @@ class CommandsAppTest < ActiveSupport::TestCase
     assert_equal "rm -r corddir", new_command.cut_cord("corddir").join(" ")
   end
 
+  test "extract assets" do
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/extracted/app-web-999", "&&",
+      :docker, :stop, "-t 1", "app-web-assets", "2> /dev/null", "|| true", "&&",
+      :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "dhh/app:latest", "sleep 1000000", "&&",
+      :docker, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/assets/extracted/app-web-999", "&&",
+      :docker, :stop, "-t 1", "app-web-assets"
+    ], new_command(asset_path: "/public/assets").extract_assets
+  end
+
+  test "sync asset volumes" do
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-999", ".kamal/assets/volumes/app-web-999"
+    ], new_command(asset_path: "/public/assets").sync_asset_volumes
+
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-999", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-999", ".kamal/assets/volumes/app-web-998", "|| true", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-998", ".kamal/assets/volumes/app-web-999", "|| true",
+    ], new_command(asset_path: "/public/assets").sync_asset_volumes(old_version: 998)
+  end
+
+  test "clean up assets" do
+    assert_equal [
+      :find, ".kamal/assets/extracted", "-maxdepth 1", "-name", "'app-web-*'", "!", "-name", "app-web-999", "-exec rm -rf \"{}\" +", ";",
+      :find, ".kamal/assets/volumes", "-maxdepth 1", "-name", "'app-web-*'", "!", "-name", "app-web-999", "-exec rm -rf \"{}\" +"
+    ], new_command(asset_path: "/public/assets").clean_up_assets
+  end
+
   private
-    def new_command(role: "web")
-      Kamal::Commands::App.new(Kamal::Configuration.new(@config, destination: @destination, version: "999"), role: role)
+    def new_command(role: "web", **additional_config)
+      config = Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999")
+      Kamal::Commands::App.new(config, role: config.role(role))
     end
 end

test/commands/builder_test.rb

@@ -37,6 +37,14 @@ class CommandsBuilderTest < ActiveSupport::TestCase
       builder.push.join(" ")
   end
 
+  test "target multiarch local when arch is set" do
+    builder = new_builder_command(builder: { "local" => { "arch" => "amd64" } })
+    assert_equal "multiarch", builder.name
+    assert_equal \
+      "docker buildx build --push --platform linux/amd64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .",
+      builder.push.join(" ")
+  end
+
   test "target native remote when only remote is set" do
     builder = new_builder_command(builder: { "remote" => { "arch" => "amd64" }, "cache" => { "type" => "gha" } })
     assert_equal "native/remote", builder.name
@@ -103,6 +111,18 @@ class CommandsBuilderTest < ActiveSupport::TestCase
       builder.push.join(" ")
   end
 
+  test "build with ssh agent socket" do
+    builder = new_builder_command(builder: { "ssh" => 'default=$SSH_AUTH_SOCK' })
+
+    assert_equal \
+      "-t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --ssh default=$SSH_AUTH_SOCK",
+      builder.target.build_options.join(" ")
+  end
+
+  test "validate image" do
+    assert_equal "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:123 | grep -x app || (echo \"Image dhh/app:123 is missing the 'service' label\" && exit 1)", new_builder_command.validate_image.join(" ")
+  end
+
   private
     def new_builder_command(additional_config = {})
       Kamal::Commands::Builder.new(Kamal::Configuration.new(@config.merge(additional_config), version: "123"))

test/commands/docker_test.rb

@@ -9,7 +9,7 @@ class CommandsDockerTest < ActiveSupport::TestCase
   end
 
   test "install" do
-    assert_equal "curl -fsSL https://get.docker.com | sh", @docker.install.join(" ")
+    assert_equal "sh -c 'curl -fsSL https://get.docker.com || wget -O - https://get.docker.com || echo \"exit 1\"' | sh", @docker.install.join(" ")
   end
 
   test "installed?" do
@@ -21,6 +21,6 @@ class CommandsDockerTest < ActiveSupport::TestCase
   end
 
   test "superuser?" do
-    assert_equal '[ "${EUID:-$(id -u)}" -eq 0 ]', @docker.superuser?.join(" ")
+    assert_equal '[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', @docker.superuser?.join(" ")
   end
 end

test/commands/healthcheck_test.rb

@@ -92,6 +92,13 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
       new_command.logs.join(" ")
   end
 
+  test "logs with custom lines number" do
+    @config[:healthcheck] = { "log_lines" => 150 }
+    assert_equal \
+      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker logs --tail 150 2>&1",
+      new_command.logs.join(" ")
+  end
+
   test "logs with destination" do
     @destination = "staging"
 

test/commands/prune_test.rb

@@ -10,7 +10,7 @@ class CommandsPruneTest < ActiveSupport::TestCase
 
   test "dangling images" do
     assert_equal \
-      "docker image prune --force --filter label=service=app --filter dangling=true",
+      "docker image prune --force --filter label=service=app",
       new_command.dangling_images.join(" ")
   end
 
@@ -20,10 +20,20 @@ class CommandsPruneTest < ActiveSupport::TestCase
       new_command.tagged_images.join(" ")
   end
 
-  test "containers" do
+  test "app containers" do
     assert_equal \
       "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done",
-      new_command.containers.join(" ")
+      new_command.app_containers(retain: 5).join(" ")
+
+    assert_equal \
+      "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +4 | while read container_id; do docker rm $container_id; done",
+      new_command.app_containers(retain: 3).join(" ")
+  end
+
+  test "healthcheck containers" do
+    assert_equal \
+      "docker container prune --force --filter label=service=healthcheck-app",
+      new_command.healthcheck_containers.join(" ")
   end
 
   private

test/commands/registry_test.rb

@@ -15,7 +15,7 @@ class CommandsRegistryTest < ActiveSupport::TestCase
 
   test "registry login" do
     assert_equal \
-      "docker login hub.docker.com -u dhh -p secret",
+      "docker login hub.docker.com -u \"dhh\" -p \"secret\"",
       @registry.login.join(" ")
   end
 
@@ -24,7 +24,18 @@ class CommandsRegistryTest < ActiveSupport::TestCase
     @config[:registry]["password"] = [ "KAMAL_REGISTRY_PASSWORD" ]
 
     assert_equal \
-      "docker login hub.docker.com -u dhh -p more-secret",
+      "docker login hub.docker.com -u \"dhh\" -p \"more-secret\"",
+      @registry.login.join(" ")
+  ensure
+    ENV.delete("KAMAL_REGISTRY_PASSWORD")
+  end
+
+  test "registry login escape password" do
+    ENV["KAMAL_REGISTRY_PASSWORD"] = "more-secret'\""
+    @config[:registry]["password"] = [ "KAMAL_REGISTRY_PASSWORD" ]
+
+    assert_equal \
+      "docker login hub.docker.com -u \"dhh\" -p \"more-secret'\\\"\"",
       @registry.login.join(" ")
   ensure
     ENV.delete("KAMAL_REGISTRY_PASSWORD")
@@ -35,7 +46,7 @@ class CommandsRegistryTest < ActiveSupport::TestCase
     @config[:registry]["username"] = [ "KAMAL_REGISTRY_USERNAME" ]
 
     assert_equal \
-      "docker login hub.docker.com -u also-secret -p secret",
+      "docker login hub.docker.com -u \"also-secret\" -p \"secret\"",
       @registry.login.join(" ")
   ensure
     ENV.delete("KAMAL_REGISTRY_USERNAME")

test/commands/traefik_test.rb

@@ -18,72 +18,72 @@ class CommandsTraefikTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["host_port"] = "8080"
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["publish"] = false
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with ports configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["options"] = {"publish" => %w[9000:9000 9001:9001]}
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with volumes configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json] }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with several options configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json], "publish" => %w[8080:8080], "memory" => "512m"}
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with labels configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["labels"] = { "traefik.http.routers.dashboard.service" => "api@internal", "traefik.http.routers.dashboard.middlewares" => "auth" }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.dashboard.service=\"api@internal\" --label traefik.http.routers.dashboard.middlewares=\"auth\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --label traefik.http.routers.dashboard.service=\"api@internal\" --label traefik.http.routers.dashboard.middlewares=\"auth\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with env configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -91,7 +91,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
     @config.delete(:traefik)
 
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"",
       new_command.run.join(" ")
   end
 
@@ -99,7 +99,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -107,7 +107,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
     @config[:traefik]["args"]["log.level"] = "ERROR"
 
     assert_equal \
-    "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"ERROR\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+    "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"ERROR\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -167,20 +167,20 @@ class CommandsTraefikTest < ActiveSupport::TestCase
 
   test "traefik follow logs" do
     assert_equal \
-      "ssh -t root@1.1.1.1 'docker logs traefik --timestamps --tail 10 --follow 2>&1'",
+      "ssh -t root@1.1.1.1 -p 22 'docker logs traefik --timestamps --tail 10 --follow 2>&1'",
       new_command.follow_logs(host: @config[:servers].first)
   end
 
   test "traefik follow logs with grep hello!" do
     assert_equal \
-      "ssh -t root@1.1.1.1 'docker logs traefik --timestamps --tail 10 --follow 2>&1 | grep \"hello!\"'",
+      "ssh -t root@1.1.1.1 -p 22 'docker logs traefik --timestamps --tail 10 --follow 2>&1 | grep \"hello!\"'",
       new_command.follow_logs(host: @config[:servers].first, grep: 'hello!')
   end
 
   test "env_file" do
     @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
 
-    assert_equal "EXAMPLE_API_KEY=456\n", new_command.env_file
+    assert_equal "EXAMPLE_API_KEY=456\n", new_command.env_file.to_s
   end
 
   test "host_env_file_path" do

test/configuration/accessory_test.rb

@@ -49,6 +49,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
           }
         },
         "monitoring" => {
+          "service" => "custom-monitoring",
           "image" => "monitoring:latest",
           "roles" => [ "web" ],
           "port" => "4321:4321",
@@ -72,6 +73,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
   test "service name" do
     assert_equal "app-mysql", @config.accessory(:mysql).service_name
     assert_equal "app-redis", @config.accessory(:redis).service_name
+    assert_equal "custom-monitoring", @config.accessory(:monitoring).service_name
   end
 
   test "port" do
@@ -123,7 +125,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
       MYSQL_ROOT_HOST=%
     ENV
 
-    assert_equal expected, @config.accessory(:mysql).env_file
+    assert_equal expected, @config.accessory(:mysql).env_file.to_s
   ensure
     ENV["MYSQL_ROOT_PASSWORD"] = nil
   end
@@ -149,10 +151,16 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
     assert_match "%", @config.accessory(:mysql).files.keys[2].read
   end
 
-  test "directories" do
+  test "directory with a relative path" do
+    @deploy[:accessories]["mysql"]["directories"] = [ "data:/var/lib/mysql" ]
     assert_equal({"$PWD/app-mysql/data"=>"/var/lib/mysql"}, @config.accessory(:mysql).directories)
   end
 
+  test "directory with an absolute path" do
+    @deploy[:accessories]["mysql"]["directories"] = [ "/var/data/mysql:/var/lib/mysql" ]
+    assert_equal({"/var/data/mysql"=>"/var/lib/mysql"}, @config.accessory(:mysql).directories)
+  end
+
   test "options" do
     assert_equal ["--cpus", "\"4\"", "--memory", "\"2GB\""], @config.accessory(:redis).option_args
   end

test/configuration/builder_test.rb

@@ -148,4 +148,14 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase
 
     assert_equal "..", @config_with_builder_option.builder.context
   end
+
+  test "ssh" do
+    assert_nil @config.builder.ssh
+  end
+
+  test "setting ssh params" do
+    @deploy_with_builder_option[:builder] = { "ssh" => 'default=$SSH_AUTH_SOCK' }
+
+    assert_equal 'default=$SSH_AUTH_SOCK', @config_with_builder_option.builder.ssh
+  end
 end

test/configuration/role_test.rb

@@ -42,7 +42,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
   end
 
   test "special label args for web" do
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.routers.app-web.priority=\"2\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args
   end
 
   test "custom labels" do
@@ -66,7 +66,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       c[:servers]["beta"] = { "traefik" => "true", "hosts" => [ "1.1.1.5" ] }
     })
 
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "traefik.http.services.app-beta.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "traefik.http.services.app-beta.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.routers.app-beta.priority=\"2\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args
   end
 
   test "env overwritten by role" do
@@ -77,7 +77,16 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected_env, @config_with_roles.role(:workers).env_file
+    assert_equal expected_env, @config_with_roles.role(:workers).env_file.to_s
+  end
+
+  test "container name" do
+    ENV["VERSION"] = "12345"
+
+    assert_equal "app-workers-12345", @config_with_roles.role(:workers).container_name
+    assert_equal "app-web-12345", @config_with_roles.role(:web).container_name
+  ensure
+    ENV.delete("VERSION")
   end
 
   test "env args" do
@@ -114,7 +123,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected, @config_with_roles.role(:workers).env_file
+    assert_equal expected, @config_with_roles.role(:workers).env_file.to_s
   ensure
     ENV["REDIS_PASSWORD"] = nil
     ENV["DB_PASSWORD"] = nil
@@ -139,7 +148,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected, @config_with_roles.role(:workers).env_file
+    assert_equal expected, @config_with_roles.role(:workers).env_file.to_s
   ensure
     ENV["DB_PASSWORD"] = nil
   end
@@ -162,7 +171,35 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected, @config_with_roles.role(:workers).env_file
+    assert_equal expected, @config_with_roles.role(:workers).env_file.to_s
+  ensure
+    ENV["REDIS_PASSWORD"] = nil
+  end
+
+  test "env overwritten by role with secrets" do
+    @deploy_with_roles[:env] = {
+      "clear" => {
+        "REDIS_URL" => "redis://a/b"
+      },
+      "secret" => [
+        "REDIS_PASSWORD"
+      ]
+    }
+
+    @deploy_with_roles[:servers]["workers"]["env"] = {
+      "clear" => {
+        "REDIS_URL" => "redis://c/d",
+      },
+    }
+
+    ENV["REDIS_PASSWORD"] = "secret456"
+
+    expected = <<~ENV
+      REDIS_PASSWORD=secret456
+      REDIS_URL=redis://c/d
+    ENV
+
+    assert_equal expected, @config_with_roles.role(:workers).env_file.to_s
   ensure
     ENV["REDIS_PASSWORD"] = nil
   end
@@ -194,4 +231,53 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
   test "cord container file" do
     assert_equal "/tmp/kamal-cord/cord", @config_with_roles.role(:web).cord_container_file
   end
+
+  test "asset path and volume args" do
+    ENV["VERSION"] = "12345"
+    assert_nil @config_with_roles.role(:web).asset_volume_args
+    assert_nil @config_with_roles.role(:workers).asset_volume_args
+    assert_nil @config_with_roles.role(:web).asset_path
+    assert_nil @config_with_roles.role(:workers).asset_path
+    assert !@config_with_roles.role(:web).assets?
+    assert !@config_with_roles.role(:workers).assets?
+
+    config_with_assets = Kamal::Configuration.new(@deploy_with_roles.dup.tap { |c|
+      c[:asset_path] = "foo"
+    })
+    assert_equal "foo", config_with_assets.role(:web).asset_path
+    assert_equal "foo", config_with_assets.role(:workers).asset_path
+    assert_equal ["--volume", "$(pwd)/.kamal/assets/volumes/app-web-12345:foo"], config_with_assets.role(:web).asset_volume_args
+    assert_nil config_with_assets.role(:workers).asset_volume_args
+    assert config_with_assets.role(:web).assets?
+    assert !config_with_assets.role(:workers).assets?
+
+    config_with_assets = Kamal::Configuration.new(@deploy_with_roles.dup.tap { |c|
+      c[:servers]["web"] = { "hosts" => [ "1.1.1.1", "1.1.1.2" ], "asset_path" => "bar" }
+    })
+    assert_equal "bar", config_with_assets.role(:web).asset_path
+    assert_nil config_with_assets.role(:workers).asset_path
+    assert_equal ["--volume", "$(pwd)/.kamal/assets/volumes/app-web-12345:bar"], config_with_assets.role(:web).asset_volume_args
+    assert_nil config_with_assets.role(:workers).asset_volume_args
+    assert config_with_assets.role(:web).assets?
+    assert !config_with_assets.role(:workers).assets?
+
+  ensure
+    ENV.delete("VERSION")
+  end
+
+  test "asset extracted path" do
+    ENV["VERSION"] = "12345"
+    assert_equal ".kamal/assets/extracted/app-web-12345", @config_with_roles.role(:web).asset_extracted_path
+    assert_equal ".kamal/assets/extracted/app-workers-12345", @config_with_roles.role(:workers).asset_extracted_path
+  ensure
+    ENV.delete("VERSION")
+  end
+
+  test "asset volume path" do
+    ENV["VERSION"] = "12345"
+    assert_equal ".kamal/assets/volumes/app-web-12345", @config_with_roles.role(:web).asset_volume_path
+    assert_equal ".kamal/assets/volumes/app-workers-12345", @config_with_roles.role(:workers).asset_volume_path
+  ensure
+    ENV.delete("VERSION")
+  end
 end

test/configuration/ssh_test.rb

@@ -22,6 +22,9 @@ class ConfigurationSshTest < ActiveSupport::TestCase
 
     config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "log_level" => "debug" }) })
     assert_equal 0, config.ssh.options[:logger].level
+
+    config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "port" => 2222 }) })
+    assert_equal 2222, config.ssh.options[:port]
   end
 
   test "ssh options with proxy host" do

test/configuration/volume_test.rb

@@ -0,0 +1,13 @@
+require "test_helper"
+
+class ConfigurationVolumeTest < ActiveSupport::TestCase
+  test "docker args absolute" do
+    volume = Kamal::Configuration::Volume.new(host_path: "/root/foo/bar", container_path: "/assets")
+    assert_equal ["--volume", "/root/foo/bar:/assets"], volume.docker_args
+  end
+
+  test "docker args relative" do
+    volume = Kamal::Configuration::Volume.new(host_path: "foo/bar", container_path: "/assets")
+    assert_equal ["--volume", "$(pwd)/foo/bar:/assets"], volume.docker_args
+  end
+end

test/configuration_test.rb

@@ -42,6 +42,16 @@ class ConfigurationTest < ActiveSupport::TestCase
     end
   end
 
+  test "service name valid" do
+    assert Kamal::Configuration.new(@deploy.tap { _1[:service] = "hey-app1_primary" }).valid?
+  end
+
+  test "service name invalid" do
+    assert_raise(ArgumentError) do
+      Kamal::Configuration.new @deploy.tap { _1[:service] = "app.com" }
+    end
+  end
+
   test "roles" do
     assert_equal %w[ web ], @config.roles.collect(&:name)
     assert_equal %w[ web workers ], @config_with_roles.roles.collect(&:name)
@@ -58,9 +68,9 @@ class ConfigurationTest < ActiveSupport::TestCase
     assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], @config_with_roles.all_hosts
   end
 
-  test "primary web host" do
-    assert_equal "1.1.1.1", @config.primary_web_host
-    assert_equal "1.1.1.1", @config_with_roles.primary_web_host
+  test "primary host" do
+    assert_equal "1.1.1.1", @config.primary_host
+    assert_equal "1.1.1.1", @config_with_roles.primary_host
   end
 
   test "traefik hosts" do
@@ -75,7 +85,7 @@ class ConfigurationTest < ActiveSupport::TestCase
   test "version no git repo" do
     ENV.delete("VERSION")
 
-    @config.expects(:system).with("git rev-parse").returns(nil)
+    Kamal::Git.expects(:used?).returns(nil)
     error = assert_raises(RuntimeError) { @config.version}
     assert_match /no git repository found/, error.message
   end
@@ -83,16 +93,16 @@ class ConfigurationTest < ActiveSupport::TestCase
   test "version from git committed" do
     ENV.delete("VERSION")
 
-    @config.expects(:`).with("git rev-parse HEAD").returns("git-version")
-    Kamal::Utils.expects(:uncommitted_changes).returns("")
+    Kamal::Git.expects(:revision).returns("git-version")
+    Kamal::Git.expects(:uncommitted_changes).returns("")
     assert_equal "git-version", @config.version
   end
 
   test "version from git uncommitted" do
     ENV.delete("VERSION")
 
-    @config.expects(:`).with("git rev-parse HEAD").returns("git-version")
-    Kamal::Utils.expects(:uncommitted_changes).returns("M   file\n")
+    Kamal::Git.expects(:revision).returns("git-version")
+    Kamal::Git.expects(:uncommitted_changes).returns("M   file\n")
     assert_match /^git-version_uncommitted_[0-9a-f]{16}$/, @config.version
   end
 
@@ -124,12 +134,8 @@ class ConfigurationTest < ActiveSupport::TestCase
     assert_equal "app-missing", @config.service_with_version
   end
 
-  test "env with missing secret" do
-    assert_raises(KeyError) do
-      config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!({
-        env: { "secret" => [ "PASSWORD" ] }
-      }) }).ensure_env_available
-    end
+  test "healthcheck service" do
+    assert_equal "healthcheck-app", @config.healthcheck_service
   end
 
   test "valid config" do
@@ -169,6 +175,16 @@ class ConfigurationTest < ActiveSupport::TestCase
     end
   end
 
+  test "allow_empty_roles" do
+    assert_silent do
+      Kamal::Configuration.new @deploy.merge(servers: { "web" => %w[ web ], "workers" => { "hosts" => %w[ ] } }, allow_empty_roles: true)
+    end
+
+    assert_raises(ArgumentError) do
+      Kamal::Configuration.new @deploy.merge(servers: { "web" => %w[], "workers" => { "hosts" => %w[] } }, allow_empty_roles: true)
+    end
+  end
+
   test "volume_args" do
     assert_equal ["--volume", "/local/path:/container/path"], @config.volume_args
   end
@@ -210,6 +226,18 @@ class ConfigurationTest < ActiveSupport::TestCase
     end
   end
 
+  test "destination required" do
+    dest_config_file = Pathname.new(File.expand_path("fixtures/deploy_for_required_dest.yml", __dir__))
+
+    assert_raises(ArgumentError) do
+      config = Kamal::Configuration.create_from config_file: dest_config_file
+    end
+
+    assert_nothing_raised do
+      config = Kamal::Configuration.create_from config_file: dest_config_file, destination: "world"
+    end
+  end
+
   test "to_h" do
     expected_config = \
       { :roles=>["web"],
@@ -219,12 +247,12 @@ class ConfigurationTest < ActiveSupport::TestCase
         :repository=>"dhh/app",
         :absolute_image=>"dhh/app:missing",
         :service_with_version=>"app-missing",
-        :ssh_options=>{ :user=>"root", :auth_methods=>["publickey"], log_level: :fatal, keepalive: true, keepalive_interval: 30 },
+        :ssh_options=>{ :user=>"root", port: 22, log_level: :fatal, keepalive: true, keepalive_interval: 30 },
         :sshkit=>{},
         :volume_args=>["--volume", "/local/path:/container/path"],
         :builder=>{},
         :logging=>["--log-opt", "max-size=\"10m\""],
-        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }}
+        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }}
 
     assert_equal expected_config, @config.to_h
   end
@@ -265,4 +293,38 @@ class ConfigurationTest < ActiveSupport::TestCase
     SecureRandom.expects(:hex).with(16).returns("09876543211234567890098765432112")
     assert_equal "09876543211234567890098765432112", @config.run_id
   end
+
+  test "asset path" do
+    assert_nil @config.asset_path
+    assert_equal "foo", Kamal::Configuration.new(@deploy.merge!(asset_path: "foo")).asset_path
+  end
+
+  test "primary role" do
+    assert_equal "web", @config.primary_role.name
+
+    config = Kamal::Configuration.new(@deploy_with_roles.deep_merge({
+      servers: { "alternate_web" => { "hosts" => [ "1.1.1.4", "1.1.1.5" ] } },
+      primary_role: "alternate_web" } ))
+
+
+    assert_equal "alternate_web", config.primary_role.name
+    assert_equal "1.1.1.4", config.primary_host
+    assert config.role(:alternate_web).primary?
+    assert config.role(:alternate_web).running_traefik?
+  end
+
+  test "primary role missing" do
+    error = assert_raises(ArgumentError) do
+      Kamal::Configuration.new(@deploy.merge(primary_role: "bar"))
+    end
+    assert_match /bar isn't defined/, error.message
+  end
+
+  test "retain_containers" do
+    assert_equal 5, @config.retain_containers
+    config = Kamal::Configuration.new(@deploy_with_roles.merge(retain_containers: 2))
+    assert_equal 2, config.retain_containers
+
+    assert_raises(ArgumentError) { Kamal::Configuration.new(@deploy_with_roles.merge(retain_containers: 0)) }
+  end
 end

test/env_file_test.rb

@@ -0,0 +1,102 @@
+require "test_helper"
+
+class EnvFileTest < ActiveSupport::TestCase
+  test "env file simple" do
+    env = {
+      "foo" => "bar",
+      "baz" => "haz"
+    }
+
+    assert_equal "foo=bar\nbaz=haz\n", \
+      Kamal::EnvFile.new(env).to_s
+  end
+
+  test "env file clear" do
+    env = {
+      "clear" => {
+        "foo" => "bar",
+        "baz" => "haz"
+      }
+    }
+
+    assert_equal "foo=bar\nbaz=haz\n", \
+      Kamal::EnvFile.new(env).to_s
+  end
+
+  test "env file empty" do
+    assert_equal "\n", Kamal::EnvFile.new({}).to_s
+  end
+
+  test "env file secret" do
+    ENV["PASSWORD"] = "hello"
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_equal "PASSWORD=hello\n", \
+      Kamal::EnvFile.new(env).to_s
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "env file secret escaped newline" do
+    ENV["PASSWORD"] = "hello\\nthere"
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_equal "PASSWORD=hello\\\\nthere\n", \
+      Kamal::EnvFile.new(env).to_s
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "env file secret newline" do
+    ENV["PASSWORD"] = "hello\nthere"
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_equal "PASSWORD=hello\\nthere\n", \
+      Kamal::EnvFile.new(env).to_s
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "env file missing secret" do
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_raises(KeyError) { Kamal::EnvFile.new(env).to_s }
+
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "env file secret and clear" do
+    ENV["PASSWORD"] = "hello"
+    env = {
+      "secret" => [ "PASSWORD" ],
+      "clear" => {
+        "foo" => "bar",
+        "baz" => "haz"
+      }
+    }
+
+    assert_equal "PASSWORD=hello\nfoo=bar\nbaz=haz\n", \
+      Kamal::EnvFile.new(env).to_s
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "stringIO conversion" do
+    env = {
+      "foo" => "bar",
+      "baz" => "haz"
+    }
+
+    assert_equal "foo=bar\nbaz=haz\n", \
+      StringIO.new(Kamal::EnvFile.new(env)).read
+  end
+end

test/fixtures/deploy_for_required_dest.world.yml

@@ -0,0 +1,5 @@
+servers:
+  - 1.1.1.1
+  - 1.1.1.2
+env:
+  REDIS_URL: redis://x/y

test/fixtures/deploy_for_required_dest.yml

@@ -0,0 +1,7 @@
+service: app
+image: dhh/app
+registry:
+  server: registry.digitalocean.com
+  username: <%= "my-user" %>
+  password: <%= "my-password" %>
+require_destination: true

test/fixtures/deploy_primary_web_role_override.yml

@@ -0,0 +1,20 @@
+service: app
+image: dhh/app
+servers:
+  web_chicago:
+    traefik: enabled
+    hosts:
+      - 1.1.1.1
+      - 1.1.1.2
+  web_tokyo:
+    traefik: enabled
+    hosts:
+      - 1.1.1.3
+      - 1.1.1.4
+env:
+  REDIS_URL: redis://x/y
+registry:
+  server: registry.digitalocean.com
+  username: user
+  password: pw
+primary_role: web_tokyo

test/fixtures/deploy_with_aliases.yml

@@ -0,0 +1,36 @@
+# helper aliases
+chicago_hosts: &chicago_hosts
+  hosts:
+    - 1.1.1.1
+    - 1.1.1.2
+tokyo_hosts: &tokyo_hosts
+  hosts:
+    - 1.1.1.3
+    - 1.1.1.4
+web_common: &web_common
+  env:
+    ROLE: "web"
+  traefik: true
+
+# actual config
+service: app
+image: dhh/app
+servers:
+  web:
+    <<: *chicago_hosts
+    <<: *web_common
+  web_tokyo:
+    <<: *tokyo_hosts
+    <<: *web_common
+  workers:
+    cmd: bin/jobs
+    <<: *chicago_hosts
+  workers_tokyo:
+    cmd: bin/jobs
+    <<: *tokyo_hosts
+env:
+  REDIS_URL: redis://x/y
+registry:
+  server: registry.digitalocean.com
+  username: user
+  password: pw

test/fixtures/deploy_with_assets.yml

@@ -0,0 +1,9 @@
+service: app
+image: dhh/app
+servers:
+  - "1.1.1.1"
+  - "1.1.1.2"
+registry:
+  username: user
+  password: pw
+asset_path: /public/assets

test/fixtures/deploy_with_low_percentage_boot_strategy.yml

@@ -0,0 +1,17 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+
+registry:
+  username: user
+  password: pw
+
+boot:
+  limit: 1%
+  wait: 2

test/fixtures/deploy_with_percentage_boot_strategy.yml

@@ -13,5 +13,5 @@ registry:
   password: pw
 
 boot:
-  limit: 25%
+  limit: 1%
   wait: 2

test/fixtures/deploy_with_remote_builder.yml

@@ -0,0 +1,41 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+registry:
+  username: user
+  password: pw
+
+accessories:
+  mysql:
+    image: mysql:5.7
+    host: 1.1.1.3
+    port: 3306
+    env:
+      clear:
+        MYSQL_ROOT_HOST: '%'
+      secret:
+        - MYSQL_ROOT_PASSWORD
+    files:
+      - test/fixtures/files/my.cnf:/etc/mysql/my.cnf
+    directories:
+      - data:/var/lib/mysql
+  redis:
+    image: redis:latest
+    roles:
+      - web
+    port: 6379
+    directories:
+      - data:/data
+
+readiness_delay: 0
+
+builder:
+  remote:
+    arch: amd64
+    host: ssh://app@1.1.1.5

test/fixtures/deploy_workers_only.yml

@@ -0,0 +1,12 @@
+service: app
+image: dhh/app
+servers:
+  workers:
+    traefik: false
+    hosts:
+      - 1.1.1.1
+      - 1.1.1.2
+primary_role: workers
+registry:
+  username: user
+  password: pw

test/git_test.rb

@@ -0,0 +1,13 @@
+require "test_helper"
+
+class GitTest < ActiveSupport::TestCase
+  test "uncommitted changes exist" do
+    Kamal::Git.expects(:`).with("git status --porcelain").returns("M   file\n")
+    assert_equal "M   file", Kamal::Git.uncommitted_changes
+  end
+
+  test "uncommitted changes do not exist" do
+    Kamal::Git.expects(:`).with("git status --porcelain").returns("")
+    assert_equal "", Kamal::Git.uncommitted_changes
+  end
+end

test/integration/app_test.rb

@@ -10,8 +10,7 @@ class AppTest < IntegrationTest
 
     kamal :app, :stop
 
-    # traefik is up and returns 404s when it can't match a route
-    assert_app_not_found
+    assert_app_is_down
 
     kamal :app, :start
 
@@ -51,7 +50,6 @@ class AppTest < IntegrationTest
 
     kamal :app, :remove
 
-    # traefik is up and returns 404s when it can't match a route
-    assert_app_not_found
+    assert_app_is_down
   end
 end

test/integration/docker/deployer/app/.kamal/hooks/docker-setup

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "Docker set up!"
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/docker-setup

test/integration/docker/deployer/app/.kamal/hooks/post-traefik-reboot

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "Rebooted Traefik on ${KAMAL_HOSTS}"
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/post-traefik-reboot

test/integration/docker/deployer/app/.kamal/hooks/pre-traefik-reboot

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "Rebooting Traefik on ${KAMAL_HOSTS}..."
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-traefik-reboot

test/integration/docker/deployer/app/Dockerfile

@@ -5,4 +5,5 @@ COPY default.conf /etc/nginx/conf.d/default.conf
 ARG COMMIT_SHA
 RUN echo $COMMIT_SHA > /usr/share/nginx/html/version
 RUN mkdir -p /usr/share/nginx/html/versions && echo "version" > /usr/share/nginx/html/versions/$COMMIT_SHA
+RUN mkdir -p /usr/share/nginx/html/versions && echo "hidden" > /usr/share/nginx/html/versions/.hidden
 

test/integration/docker/deployer/app/config/deploy.yml

@@ -24,9 +24,10 @@ traefik:
   args:
     accesslog: true
     accesslog.format: json
-  image: registry:4443/traefik:v2.9
+  image: registry:4443/traefik:v2.10
 accessories:
   busybox:
+    service: custom-busybox
     image: registry:4443/busybox:1.36.0
     cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done'
     roles:

test/integration/docker/deployer/setup.sh

@@ -19,5 +19,9 @@ push_image_to_registry_4443() {
 
 install_kamal
 push_image_to_registry_4443 nginx 1-alpine-slim
-push_image_to_registry_4443 traefik v2.9
+push_image_to_registry_4443 traefik v2.10
 push_image_to_registry_4443 busybox 1.36.0
+
+# .ssh is on a shared volume that persists between runs. Clean it up as the
+# churn of temporary vm IPs can eventually create conflicts.
+rm -f /root/.ssh/known_hosts

test/integration/docker/shared/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.10
+FROM ubuntu:22.04
 
 WORKDIR /work
 

test/integration/docker/vm/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.10
+FROM ubuntu:22.04
 
 WORKDIR /work
 

test/integration/integration_test.rb

@@ -55,12 +55,6 @@ class IntegrationTest < ActiveSupport::TestCase
       assert_app_version(version, response) if version
     end
 
-    def assert_app_not_found
-      response = app_response
-      debug_response_code(response, "404")
-      assert_equal "404", response.code
-    end
-
     def wait_for_app_to_be_up(timeout: 20, up_count: 3)
       timeout_at = Time.now + timeout
       up_times = 0
@@ -109,7 +103,7 @@ class IntegrationTest < ActiveSupport::TestCase
       assert_equal "200", code
     end
 
-    def wait_for_healthy(timeout: 20)
+    def wait_for_healthy(timeout: 30)
       timeout_at = Time.now + timeout
       while docker_compose("ps -a | tail -n +2 | grep -v '(healthy)' | wc -l", capture: true) != "0"
         if timeout_at < Time.now

test/integration/lock_test.rb

@@ -10,7 +10,7 @@ class LockTest < IntegrationTest
     assert_match /Locked by: Deployer at .*\nVersion: #{latest_app_version}\nMessage: Integration Tests/m, status
 
     error = kamal :deploy, capture: true, raise_on_error: false
-    assert_match /Deploy lock found/m, error
+    assert_match /Deploy lock found. Run 'kamal lock help' for more information/m, error
 
     kamal :lock, :release
 

test/integration/main_test.rb

@@ -5,6 +5,7 @@ class MainTest < IntegrationTest
     kamal :envify
     assert_local_env_file "SECRET_TOKEN=1234"
     assert_remote_env_file "SECRET_TOKEN=1234\nCLEAR_TOKEN=4321"
+    remove_local_env_file
 
     first_version = latest_app_version
 
@@ -31,7 +32,7 @@ class MainTest < IntegrationTest
     assert_match /Traefik Host: vm2/, details
     assert_match /App Host: vm1/, details
     assert_match /App Host: vm2/, details
-    assert_match /traefik:v2.9/, details
+    assert_match /traefik:v2.10/, details
     assert_match /registry:4443\/app:#{first_version}/, details
 
     audit = kamal :audit, capture: true
@@ -53,10 +54,23 @@ class MainTest < IntegrationTest
     assert_equal "registry:4443/app:#{version}", config[:absolute_image]
     assert_equal "app-#{version}", config[:service_with_version]
     assert_equal [], config[:volume_args]
-    assert_equal({ user: "root", auth_methods: [ "publickey" ], keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])
+    assert_equal({ user: "root", port: 22, keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])
     assert_equal({ "multiarch" => false, "args" => { "COMMIT_SHA" => version } }, config[:builder])
     assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging]
-    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord"=>"/tmp/kamal-cord", "cmd"=>"wget -qO- http://localhost > /dev/null || exit 1" }, config[:healthcheck])
+    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord"=>"/tmp/kamal-cord", "log_lines" => 50, "cmd"=>"wget -qO- http://localhost > /dev/null || exit 1" }, config[:healthcheck])
+  end
+
+  test "setup and remove" do
+    # Check remove completes when nothing has been setup yet
+    kamal :remove, "-y"
+    assert_no_images_or_containers
+
+    kamal :envify
+    kamal :setup
+    assert_images_and_containers
+
+    kamal :remove, "-y"
+    assert_no_images_or_containers
   end
 
   private
@@ -64,6 +78,10 @@ class MainTest < IntegrationTest
       assert_equal contents, deployer_exec("cat .env", capture: true)
     end
 
+    def remove_local_env_file
+      deployer_exec("rm .env")
+    end
+
     def assert_remote_env_file(contents)
       assert_equal contents, docker_compose("exec vm1 cat /root/.kamal/env/roles/app-web.env", capture: true)
     end
@@ -76,5 +94,25 @@ class MainTest < IntegrationTest
       versions.each do |version|
         assert_equal "200", Net::HTTP.get_response(URI.parse("http://localhost:12345/versions/#{version}")).code
       end
+
+      assert_equal "200", Net::HTTP.get_response(URI.parse("http://localhost:12345/versions/.hidden")).code
+    end
+
+    def vm1_image_ids
+      docker_compose("exec vm1 docker image ls -q", capture: true).strip.split("\n")
+    end
+
+    def vm1_container_ids
+      docker_compose("exec vm1 docker ps -a -q", capture: true).strip.split("\n")
+    end
+
+    def assert_no_images_or_containers
+      assert vm1_image_ids.empty?
+      assert vm1_container_ids.empty?
+    end
+
+    def assert_images_and_containers
+      assert vm1_image_ids.any?
+      assert vm1_container_ids.any?
     end
 end

test/integration/traefik_test.rb

@@ -7,8 +7,19 @@ class TraefikTest < IntegrationTest
     kamal :traefik, :boot
     assert_traefik_running
 
-    kamal :traefik, :reboot
+    output = kamal :traefik, :reboot, capture: true
     assert_traefik_running
+    assert_hooks_ran "pre-traefik-reboot", "post-traefik-reboot"
+    assert_match /Rebooting Traefik on vm1,vm2.../, output
+    assert_match /Rebooted Traefik on vm1,vm2/, output
+
+    output = kamal :traefik, :reboot, :"--rolling", capture: true
+    assert_traefik_running
+    assert_hooks_ran "pre-traefik-reboot", "post-traefik-reboot"
+    assert_match /Rebooting Traefik on vm1.../, output
+    assert_match /Rebooted Traefik on vm1/, output
+    assert_match /Rebooting Traefik on vm2.../, output
+    assert_match /Rebooted Traefik on vm2/, output
 
     kamal :traefik, :boot
     assert_traefik_running
@@ -41,11 +52,11 @@ class TraefikTest < IntegrationTest
 
   private
     def assert_traefik_running
-      assert_match /traefik:v2.9   "\/entrypoint.sh/, traefik_details
+      assert_match /traefik:v2.10   "\/entrypoint.sh/, traefik_details
     end
 
     def assert_traefik_not_running
-      refute_match /traefik:v2.9   "\/entrypoint.sh/, traefik_details
+      refute_match /traefik:v2.10   "\/entrypoint.sh/, traefik_details
     end
 
     def traefik_details

test/utils_test.rb

@@ -11,67 +11,6 @@ class UtilsTest < ActiveSupport::TestCase
       Kamal::Utils.argumentize("--label", { foo: "bar" }, sensitive: true).last
   end
 
-  test "env file simple" do
-    env = {
-      "foo" => "bar",
-      "baz" => "haz"
-    }
-
-    assert_equal "foo=bar\nbaz=haz\n", \
-      Kamal::Utils.env_file_with_secrets(env)
-  end
-
-  test "env file clear" do
-    env = {
-      "clear" => {
-        "foo" => "bar",
-        "baz" => "haz"
-      }
-    }
-
-    assert_equal "foo=bar\nbaz=haz\n", \
-      Kamal::Utils.env_file_with_secrets(env)
-  end
-
-  test "env file secret" do
-    ENV["PASSWORD"] = "hello"
-    env = {
-      "secret" => [ "PASSWORD" ]
-    }
-
-    assert_equal "PASSWORD=hello\n", \
-      Kamal::Utils.env_file_with_secrets(env)
-  ensure
-    ENV.delete "PASSWORD"
-  end
-
-  test "env file missing secret" do
-    env = {
-      "secret" => [ "PASSWORD" ]
-    }
-
-    assert_raises(KeyError) { Kamal::Utils.env_file_with_secrets(env) }
-
-  ensure
-    ENV.delete "PASSWORD"
-  end
-
-  test "env file secret and clear" do
-    ENV["PASSWORD"] = "hello"
-    env = {
-      "secret" => [ "PASSWORD" ],
-      "clear" => {
-        "foo" => "bar",
-        "baz" => "haz"
-      }
-    }
-
-    assert_equal "PASSWORD=hello\nfoo=bar\nbaz=haz\n", \
-      Kamal::Utils.env_file_with_secrets(env)
-  ensure
-    ENV.delete "PASSWORD"
-  end
-
   test "optionize" do
     assert_equal [ "--foo", "\"bar\"", "--baz", "\"qux\"", "--quux" ], \
       Kamal::Utils.optionize({ foo: "bar", baz: "qux", quux: true })
@@ -113,14 +52,4 @@ class UtilsTest < ActiveSupport::TestCase
     assert_equal "\"https://example.com/\\$2\"",
       Kamal::Utils.escape_shell_value("https://example.com/$2")
   end
-
-  test "uncommitted changes exist" do
-    Kamal::Utils.expects(:`).with("git status --porcelain").returns("M   file\n")
-    assert_equal "M   file", Kamal::Utils.uncommitted_changes
-  end
-
-  test "uncommitted changes do not exist" do
-    Kamal::Utils.expects(:`).with("git status --porcelain").returns("")
-    assert_equal "", Kamal::Utils.uncommitted_changes
-  end
 end