Merge branch 'main' into auto-push-env

Add a simple validation to the service name to prevent setup issues

.github/workflows/ci.yml

@@ -1,5 +1,5 @@
 name: CI
-on: 
+on:
   push:
     branches:
       - main
@@ -12,17 +12,29 @@ jobs:
           - "2.7"
           - "3.1"
           - "3.2"
+          - "3.3"
         gemfile:
           - Gemfile
+          - gemfiles/ruby_2.7.gemfile
           - gemfiles/rails_edge.gemfile
-        continue-on-error: [false]
+        exclude:
+          - ruby-version: "2.7"
+            gemfile: Gemfile
+          - ruby-version: "2.7"
+            gemfile: gemfiles/rails_edge.gemfile
+          - ruby-version: "3.1"
+            gemfile: gemfiles/ruby_2.7.gemfile
+          - ruby-version: "3.2"
+            gemfile: gemfiles/ruby_2.7.gemfile
+          - ruby-version: "3.3"
+            gemfile: gemfiles/ruby_2.7.gemfile
     name: ${{ format('Tests (Ruby {0})', matrix.ruby-version) }}
     runs-on: ubuntu-latest
-    continue-on-error: ${{ matrix.continue-on-error }}
+    continue-on-error: true
     env:
       BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install Ruby
         uses: ruby/setup-ruby@v1

Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    kamal (0.16.1)
+    kamal (1.3.1)
       activesupport (>= 7.0)
+      base64 (~> 0.2)
       bcrypt_pbkdf (~> 1.0)
       concurrent-ruby (~> 1.2)
       dotenv (~> 2.8)
@@ -15,82 +16,111 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.0.4.3)
+    actionpack (7.1.2)
-      actionview (= 7.0.4.3)
+      actionview (= 7.1.2)
-      activesupport (= 7.0.4.3)
+      activesupport (= 7.1.2)
-      rack (~> 2.0, >= 2.2.0)
+      nokogiri (>= 1.8.5)
+      racc
+      rack (>= 2.2.4)
+      rack-session (>= 1.0.1)
       rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.0)
+      rails-dom-testing (~> 2.2)
-      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+      rails-html-sanitizer (~> 1.6)
-    actionview (7.0.4.3)
+    actionview (7.1.2)
-      activesupport (= 7.0.4.3)
+      activesupport (= 7.1.2)
       builder (~> 3.1)
-      erubi (~> 1.4)
+      erubi (~> 1.11)
-      rails-dom-testing (~> 2.0)
+      rails-dom-testing (~> 2.2)
-      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+      rails-html-sanitizer (~> 1.6)
-    activesupport (7.0.4.3)
+    activesupport (7.1.2)
+      base64
+      bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
+      mutex_m
       tzinfo (~> 2.0)
+    base64 (0.2.0)
     bcrypt_pbkdf (1.1.0)
+    bigdecimal (3.1.5)
     builder (3.2.4)
     concurrent-ruby (1.2.2)
+    connection_pool (2.4.1)
     crass (1.0.6)
-    debug (1.7.2)
+    debug (1.9.1)
-      irb (>= 1.5.0)
+      irb (~> 1.10)
-      reline (>= 0.3.1)
+      reline (>= 0.3.8)
     dotenv (2.8.1)
+    drb (2.2.0)
+      ruby2_keywords
     ed25519 (1.3.0)
     erubi (1.12.0)
-    i18n (1.12.0)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    io-console (0.6.0)
+    io-console (0.7.1)
-    irb (1.6.3)
+    irb (1.11.0)
-      reline (>= 0.3.0)
+      rdoc
-    loofah (2.20.0)
+      reline (>= 0.3.8)
+    loofah (2.22.0)
       crass (~> 1.0.2)
-      nokogiri (>= 1.5.9)
+      nokogiri (>= 1.12.0)
-    method_source (1.0.0)
+    minitest (5.20.0)
-    minitest (5.18.0)
+    mocha (2.1.0)
-    mocha (2.0.2)
       ruby2_keywords (>= 0.0.5)
+    mutex_m (0.2.0)
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
-    net-ssh (7.1.0)
+    net-ssh (7.2.1)
-    nokogiri (1.14.2-arm64-darwin)
+    nokogiri (1.16.0-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.2-x86_64-darwin)
+    nokogiri (1.16.0-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.2-x86_64-linux)
+    nokogiri (1.16.0-x86_64-linux)
       racc (~> 1.4)
-    racc (1.6.2)
+    psych (5.1.2)
-    rack (2.2.6.4)
+      stringio
+    racc (1.7.3)
+    rack (3.0.8)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
     rack-test (2.1.0)
       rack (>= 1.3)
-    rails-dom-testing (2.0.3)
+    rackup (2.1.0)
-      activesupport (>= 4.2.0)
+      rack (>= 3)
+      webrick (~> 1.8)
+    rails-dom-testing (2.2.0)
+      activesupport (>= 5.0.0)
+      minitest
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.5.0)
+    rails-html-sanitizer (1.6.0)
-      loofah (~> 2.19, >= 2.19.1)
+      loofah (~> 2.21)
-    railties (7.0.4.3)
+      nokogiri (~> 1.14)
-      actionpack (= 7.0.4.3)
+    railties (7.1.2)
-      activesupport (= 7.0.4.3)
+      actionpack (= 7.1.2)
-      method_source
+      activesupport (= 7.1.2)
+      irb
+      rackup (>= 1.0.0)
       rake (>= 12.2)
-      thor (~> 1.0)
+      thor (~> 1.0, >= 1.2.2)
-      zeitwerk (~> 2.5)
+      zeitwerk (~> 2.6)
-    rake (13.0.6)
+    rake (13.1.0)
-    reline (0.3.3)
+    rdoc (6.6.2)
+      psych (>= 4.0.0)
+    reline (0.4.2)
       io-console (~> 0.5)
     ruby2_keywords (0.0.5)
-    sshkit (1.21.4)
+    sshkit (1.21.7)
+      mutex_m
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
-    thor (1.2.1)
+    stringio (3.1.0)
+    thor (1.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.7)
+    webrick (1.8.1)
+    zeitwerk (2.6.12)
 
 PLATFORMS
   arm64-darwin

README.md

@@ -1,6 +1,6 @@
 # Kamal: Deploy web apps anywhere
 
-From bare metal to cloud VMs using Docker, deploy web apps anywhere with zero downtime. Kamal uses the dynamic reverse-proxy Traefik to hold requests, while the new app container is started and the old one is stopped — working seamlessly across multiple hosts, using SSHKit to execute commands. Originally built for Rails apps, Kamal will work with any type of web app that can be containerized with Docker.
+From bare metal to cloud VMs, deploy web apps anywhere with zero downtime. Kamal has the dynamic reverse-proxy Traefik hold requests while a new app container is started and the old one is stopped. Works seamlessly across multiple hosts, using SSHKit to execute commands. Originally built for Rails apps, Kamal will work with any type of web app that can be containerized with Docker.
 
 ➡️ See [kamal-deploy.org](https://kamal-deploy.org) for documentation on [installation](https://kamal-deploy.org/docs/installation), [configuration](https://kamal-deploy.org/docs/configuration), and [commands](https://kamal-deploy.org/docs/commands).
 

gemfiles/ruby_2.7.gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+gemspec path: "../"
+
+gem "nokogiri", "~> 1.15.0"

kamal.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "ed25519", "~> 1.2"
   spec.add_dependency "bcrypt_pbkdf", "~> 1.0"
   spec.add_dependency "concurrent-ruby", "~> 1.2"
+  spec.add_dependency "base64", "~> 0.2"
 
   spec.add_development_dependency "debug"
   spec.add_development_dependency "mocha"

lib/kamal/cli/accessory.rb

@@ -5,11 +5,11 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
       if name == "all"
         KAMAL.accessory_names.each { |accessory_name| boot(accessory_name) }
       else
-        with_accessory(name) do |accessory|
+        with_accessory(name) do |accessory, hosts|
           directories(name)
           upload(name)
 
-          on(accessory.hosts) do
+          on(hosts) do
             execute *KAMAL.registry.login if login
             execute *KAMAL.auditor.record("Booted #{name} accessory"), verbosity: :debug
             execute *accessory.run
@@ -22,8 +22,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "upload [NAME]", "Upload accessory files to host", hide: true
   def upload(name)
     mutating do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           accessory.files.each do |(local, remote)|
             accessory.ensure_local_file_present(local)
 
@@ -39,8 +39,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "directories [NAME]", "Create accessory directories on host", hide: true
   def directories(name)
     mutating do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           accessory.directories.keys.each do |host_path|
             execute *accessory.make_directory(host_path)
           end
@@ -49,17 +49,21 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     end
   end
 
-  desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container)"
+  desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container; use NAME=all to boot all accessories)"
   def reboot(name)
     mutating do
-      with_accessory(name) do |accessory|
+      if name == "all"
-        on(accessory.hosts) do
+        KAMAL.accessory_names.each { |accessory_name| reboot(accessory_name) }
-          execute *KAMAL.registry.login
+      else
-        end
+        with_accessory(name) do |accessory, hosts|
+          on(hosts) do
+            execute *KAMAL.registry.login
+          end
 
-        stop(name)
+          stop(name)
-        remove_container(name)
+          remove_container(name)
-        boot(name, login: false)
+          boot(name, login: false)
+        end
       end
     end
   end
@@ -67,8 +71,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "start [NAME]", "Start existing accessory container on host"
   def start(name)
     mutating do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Started #{name} accessory"), verbosity: :debug
           execute *accessory.start
         end
@@ -79,8 +83,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "stop [NAME]", "Stop existing accessory container on host"
   def stop(name)
     mutating do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Stopped #{name} accessory"), verbosity: :debug
           execute *accessory.stop, raise_on_non_zero_exit: false
         end
@@ -103,8 +107,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     if name == "all"
       KAMAL.accessory_names.each { |accessory_name| details(accessory_name) }
     else
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) { puts capture_with_info(*accessory.info) }
+        on(hosts) { puts capture_with_info(*accessory.info) }
       end
     end
   end
@@ -113,7 +117,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
   option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
   def exec(name, cmd)
-    with_accessory(name) do |accessory|
+    with_accessory(name) do |accessory, hosts|
       case
       when options[:interactive] && options[:reuse]
         say "Launching interactive command with via SSH from existing container...", :magenta
@@ -125,14 +129,14 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
       when options[:reuse]
         say "Launching command from existing container...", :magenta
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
           capture_with_info(*accessory.execute_in_existing_container(cmd))
         end
 
       else
         say "Launching command from new container...", :magenta
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
           capture_with_info(*accessory.execute_in_new_container(cmd))
         end
@@ -146,12 +150,12 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
   option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
   def logs(name)
-    with_accessory(name) do |accessory|
+    with_accessory(name) do |accessory, hosts|
       grep = options[:grep]
 
       if options[:follow]
         run_locally do
-          info "Following logs on #{accessory.hosts}..."
+          info "Following logs on #{hosts}..."
           info accessory.follow_logs(grep: grep)
           exec accessory.follow_logs(grep: grep)
         end
@@ -159,7 +163,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
         since = options[:since]
         lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
-        on(accessory.hosts) do
+        on(hosts) do
           puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep))
         end
       end
@@ -188,8 +192,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "remove_container [NAME]", "Remove accessory container from host", hide: true
   def remove_container(name)
     mutating do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Remove #{name} accessory container"), verbosity: :debug
           execute *accessory.remove_container
         end
@@ -200,8 +204,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "remove_image [NAME]", "Remove accessory image from host", hide: true
   def remove_image(name)
     mutating do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Removed #{name} accessory image"), verbosity: :debug
           execute *accessory.remove_image
         end
@@ -212,8 +216,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "remove_service_directory [NAME]", "Remove accessory directory used for uploaded files and data directories from host", hide: true
   def remove_service_directory(name)
     mutating do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *accessory.remove_service_directory
         end
       end
@@ -223,7 +227,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   private
     def with_accessory(name)
       if accessory = KAMAL.accessory(name)
-        yield accessory
+        yield accessory, accessory_hosts(accessory)
       else
         error_on_missing_accessory(name)
       end
@@ -236,4 +240,12 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
         "No accessory by the name of '#{name}'" +
         (options ? " (options: #{options.to_sentence})" : "")
     end
+
+    def accessory_hosts(accessory)
+      if KAMAL.specific_hosts&.any?
+        KAMAL.specific_hosts & accessory.hosts
+      else
+        accessory.hosts
+      end
+    end
 end

lib/kamal/cli/app.rb

@@ -9,20 +9,26 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
           on(KAMAL.hosts) do
             execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
-            execute *KAMAL.app.tag_current_as_latest
+            execute *KAMAL.app.tag_current_image_as_latest
+
+            KAMAL.roles_on(host).each do |role|
+              app = KAMAL.app(role: role)
+              role_config = KAMAL.config.role(role)
+
+              if role_config.assets?
+                execute *app.extract_assets
+                old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+                execute *app.sync_asset_volumes(old_version: old_version)
+              end
+            end
           end
 
           on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
-            roles = KAMAL.roles_on(host)
+            KAMAL.roles_on(host).each do |role|
-
-            roles.each do |role|
               app = KAMAL.app(role: role)
               auditor = KAMAL.auditor(role: role)
               role_config = KAMAL.config.role(role)
 
-              execute *app.extract_assets if role_config.assets?
-
-
               if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
                 tmp_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
                 info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
@@ -31,9 +37,6 @@ class Kamal::Cli::App < Kamal::Cli::Base
               end
 
               old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-              original_old_version = old_version.gsub(/_replaced_[a-f0-9]{16}$/, "")
-
-              execute *app.sync_asset_volumes(old_version: original_old_version) if role_config.assets?
 
               execute *app.tie_cord(role_config.cord_host_file) if role_config.uses_cord?
 
@@ -41,20 +44,20 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
               execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
 
-              Kamal::Utils::HealthcheckPoller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+              Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
 
               if old_version.present?
                 if role_config.uses_cord?
                   cord = capture_with_info(*app.cord(version: old_version), raise_on_non_zero_exit: false).strip
                   if cord.present?
                     execute *app.cut_cord(cord)
-                    Kamal::Utils::HealthcheckPoller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
+                    Kamal::Cli::Healthcheck::Poller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
                   end
                 end
 
                 execute *app.stop(version: old_version), raise_on_non_zero_exit: false
 
-                execute *app.cleanup_assets if role_config.assets?
+                execute *app.clean_up_assets if role_config.assets?
               end
             end
           end
@@ -112,14 +115,16 @@ class Kamal::Cli::App < Kamal::Cli::Base
       say "Get current version of running container...", :magenta unless options[:version]
       using_version(options[:version] || current_running_version) do |version|
         say "Launching interactive command with version #{version} via SSH from existing container on #{KAMAL.primary_host}...", :magenta
-        run_locally { exec KAMAL.app(role: "web").execute_in_existing_container_over_ssh(cmd, host: KAMAL.primary_host) }
+        run_locally { exec KAMAL.app(role: KAMAL.primary_role).execute_in_existing_container_over_ssh(cmd, host: KAMAL.primary_host) }
       end
 
     when options[:interactive]
       say "Get most recent version available as an image...", :magenta unless options[:version]
       using_version(version_or_latest) do |version|
         say "Launching interactive command with version #{version} via SSH from new container on #{KAMAL.primary_host}...", :magenta
-        run_locally { exec KAMAL.app(role: KAMAL.roles_on(KAMAL.primary_host).first).execute_in_new_container_over_ssh(cmd, host: KAMAL.primary_host) }
+        run_locally do
+          exec KAMAL.app(role: KAMAL.primary_role).execute_in_new_container_over_ssh(cmd, host: KAMAL.primary_host)
+        end
       end
 
     when options[:reuse]
@@ -142,8 +147,12 @@ class Kamal::Cli::App < Kamal::Cli::Base
       using_version(version_or_latest) do |version|
         say "Launching command with version #{version} from new container...", :magenta
         on(KAMAL.hosts) do |host|
-          execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
+          roles = KAMAL.roles_on(host)
-          puts_by_host host, capture_with_info(*KAMAL.app.execute_in_new_container(cmd))
+
+          roles.each do |role|
+            execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_new_container(cmd))
+          end
         end
       end
     end

lib/kamal/cli/base.rb

@@ -14,8 +14,8 @@ module Kamal::Cli
     class_option :version, desc: "Run commands against a specific app version"
 
     class_option :primary, type: :boolean, aliases: "-p", desc: "Run commands only on primary host instead of all"
-    class_option :hosts, aliases: "-h", desc: "Run commands on these hosts instead of all (separate by comma)"
+    class_option :hosts, aliases: "-h", desc: "Run commands on these hosts instead of all (separate by comma, supports wildcards with *)"
-    class_option :roles, aliases: "-r", desc: "Run commands on these roles instead of all (separate by comma)"
+    class_option :roles, aliases: "-r", desc: "Run commands on these roles instead of all (separate by comma, supports wildcards with *)"
 
     class_option :config_file, aliases: "-c", default: "config/deploy.yml", desc: "Path to config file"
     class_option :destination, aliases: "-d", desc: "Specify destination to be used for config file (staging -> deploy.staging.yml)"
@@ -24,6 +24,7 @@ module Kamal::Cli
 
     def initialize(*)
       super
+      @original_env = ENV.to_h.dup
       load_envs
       initialize_commander(options_with_subcommand_class_options)
     end
@@ -37,6 +38,12 @@ module Kamal::Cli
         end
       end
 
+      def reload_envs
+        ENV.clear
+        ENV.update(@original_env)
+        load_envs
+      end
+
       def options_with_subcommand_class_options
         options.merge(@_initializer.last[:class_options] || {})
       end
@@ -75,8 +82,6 @@ module Kamal::Cli
       def mutating
         return yield if KAMAL.holding_lock?
 
-        KAMAL.config.ensure_env_available
-
         run_hook "pre-connect"
 
         ensure_run_directory

lib/kamal/cli/build.rb

@@ -1,3 +1,5 @@
+require "uri"
+
 class Kamal::Cli::Build < Kamal::Cli::Base
   class BuildError < StandardError; end
 
@@ -17,7 +19,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base
       verify_local_dependencies
       run_hook "pre-build"
 
-      if (uncommitted_changes = Kamal::Utils.uncommitted_changes).present?
+      if (uncommitted_changes = Kamal::Git.uncommitted_changes).present?
         say "The following paths have uncommitted changes:\n #{uncommitted_changes}", :yellow
       end
 
@@ -48,6 +50,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base
         execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
         execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
         execute *KAMAL.builder.pull
+        execute *KAMAL.builder.validate_image
       end
     end
   end
@@ -55,6 +58,10 @@ class Kamal::Cli::Build < Kamal::Cli::Base
   desc "create", "Create a build setup"
   def create
     mutating do
+      if (remote_host = KAMAL.config.builder.remote_host)
+        connect_to_remote_host(remote_host)
+      end
+
       run_locally do
         begin
           debug "Using builder: #{KAMAL.builder.name}"
@@ -103,4 +110,14 @@ class Kamal::Cli::Build < Kamal::Cli::Base
         end
       end
     end
+
+    def connect_to_remote_host(remote_host)
+      remote_uri = URI.parse(remote_host)
+      if remote_uri.scheme == "ssh"
+        options = { user: remote_uri.user, port: remote_uri.port }.compact
+        on(remote_uri.host, options) do
+          execute "true"
+        end
+      end
+    end
 end

lib/kamal/cli/env.rb

@@ -1,50 +1,61 @@
 require "tempfile"
 
 class Kamal::Cli::Env < Kamal::Cli::Base
-  desc "push", "Push the env file to the remote hosts"
+  desc "push", "Push the env files to the remote hosts"
+  option :env_type, type: :string, desc: "Type of env files", enum: %w[secret clear all], default: "all"
   def push
+    secret = %w[secret all].include?(options[:env_type])
+    clear = %w[clear all].include?(options[:env_type])
+
     mutating do
       on(KAMAL.hosts) do
+        execute *KAMAL.auditor.record("Pushed env files"), verbosity: :debug
+
         KAMAL.roles_on(host).each do |role|
           role_config = KAMAL.config.role(role)
           execute *KAMAL.app(role: role).make_env_directory
-          upload! StringIO.new(role_config.env_file), role_config.host_env_file_path, mode: 400
+          upload! StringIO.new(role_config.env_file.secret), role_config.host_secret_env_file_path, mode: 400 if secret
+          upload! StringIO.new(role_config.env_file.clear), role_config.host_clear_env_file_path, mode: 400 if clear
         end
       end
 
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.traefik.make_env_directory
-        upload! StringIO.new(KAMAL.traefik.env_file), KAMAL.traefik.host_env_file_path, mode: 400
+        upload! StringIO.new(KAMAL.traefik.env_file.secret), KAMAL.traefik.host_secret_env_file_path, mode: 400 if secret
+        upload! StringIO.new(KAMAL.traefik.env_file.clear), KAMAL.traefik.host_clear_env_file_path, mode: 400 if clear
       end
 
       on(KAMAL.accessory_hosts) do
         KAMAL.accessories_on(host).each do |accessory|
           accessory_config = KAMAL.config.accessory(accessory)
           execute *KAMAL.accessory(accessory).make_env_directory
-          upload! StringIO.new(accessory_config.env_file), accessory_config.host_env_file_path, mode: 400
+          upload! StringIO.new(accessory_config.env_file.secret), accessory_config.host_secret_env_file_path, mode: 400 if secret
+          upload! StringIO.new(accessory_config.env_file.clear), accessory_config.host_clear_env_file_path, mode: 400 if clear
         end
       end
     end
   end
 
-  desc "delete", "Delete the env file from the remote hosts"
+  desc "delete", "Delete the env files from the remote hosts"
   def delete
     mutating do
       on(KAMAL.hosts) do
+        execute *KAMAL.auditor.record("Deleted env files"), verbosity: :debug
+
         KAMAL.roles_on(host).each do |role|
           role_config = KAMAL.config.role(role)
-          execute *KAMAL.app(role: role).remove_env_file
+          execute *KAMAL.app(role: role).remove_env_files
         end
       end
 
       on(KAMAL.traefik_hosts) do
-        execute *KAMAL.traefik.remove_env_file
+        execute *KAMAL.traefik.remove_env_files
       end
 
       on(KAMAL.accessory_hosts) do
         KAMAL.accessories_on(host).each do |accessory|
           accessory_config = KAMAL.config.accessory(accessory)
-          execute *KAMAL.accessory(accessory).remove_env_file
+          execute *KAMAL.accessory(accessory).remove_env_files
         end
       end
     end

lib/kamal/cli/healthcheck.rb

@@ -3,11 +3,12 @@ class Kamal::Cli::Healthcheck < Kamal::Cli::Base
 
   desc "perform", "Health check current app version"
   def perform
+    raise "The primary host is not configured to run Traefik" unless KAMAL.config.role(KAMAL.config.primary_role).running_traefik?
     on(KAMAL.primary_host) do
       begin
         execute *KAMAL.healthcheck.run
-        Kamal::Utils::HealthcheckPoller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
+        Poller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
-      rescue Kamal::Utils::HealthcheckPoller::HealthcheckError => e
+      rescue Poller::HealthcheckError => e
         error capture_with_info(*KAMAL.healthcheck.logs)
         error capture_with_pretty_json(*KAMAL.healthcheck.container_health_log)
         raise

lib/kamal/cli/healthcheck/poller.rb

@@ -0,0 +1,64 @@
+module Kamal::Cli::Healthcheck::Poller
+  extend self
+
+  TRAEFIK_UPDATE_DELAY = 5
+
+  class HealthcheckError < StandardError; end
+
+  def wait_for_healthy(pause_after_ready: false, &block)
+    attempt = 1
+    max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+    begin
+      case status = block.call
+      when "healthy"
+        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
+      when "running" # No health check configured
+        sleep KAMAL.config.readiness_delay if pause_after_ready
+      else
+        raise HealthcheckError, "container not ready (#{status})"
+      end
+    rescue HealthcheckError => e
+      if attempt <= max_attempts
+        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+        sleep attempt
+        attempt += 1
+        retry
+      else
+        raise
+      end
+    end
+
+    info "Container is healthy!"
+  end
+
+  def wait_for_unhealthy(pause_after_ready: false, &block)
+    attempt = 1
+    max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+    begin
+      case status = block.call
+      when "unhealthy"
+        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
+      else
+        raise HealthcheckError, "container not unhealthy (#{status})"
+      end
+    rescue HealthcheckError => e
+      if attempt <= max_attempts
+        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+        sleep attempt
+        attempt += 1
+        retry
+      else
+        raise
+      end
+    end
+
+    info "Container is unhealthy!"
+  end
+
+  private
+    def info(message)
+      SSHKit.config.output.info(message)
+    end
+end

lib/kamal/cli/main.rb

@@ -1,9 +1,14 @@
 class Kamal::Cli::Main < Kamal::Cli::Base
-  desc "setup", "Setup all accessories and deploy app to servers"
+  desc "setup", "Setup all accessories, push the env, and deploy app to servers"
   def setup
     print_runtime do
       mutating do
+        say "Ensure Docker is installed...", :magenta
         invoke "kamal:cli:server:bootstrap"
+
+        say "Push env files...", :magenta
+        invoke "kamal:cli:env:push"
+
         invoke "kamal:cli:accessory:boot", [ "all" ]
         deploy
       end
@@ -30,14 +35,18 @@ class Kamal::Cli::Main < Kamal::Cli::Base
 
         run_hook "pre-deploy"
 
+        push_env(invoke_options)
+
         say "Ensure Traefik is running...", :magenta
         invoke "kamal:cli:traefik:boot", [], invoke_options
 
-        say "Ensure app can pass healthcheck...", :magenta
+        if KAMAL.config.role(KAMAL.config.primary_role).running_traefik?
-        invoke "kamal:cli:healthcheck:perform", [], invoke_options
+          say "Ensure app can pass healthcheck...", :magenta
+          invoke "kamal:cli:healthcheck:perform", [], invoke_options
+        end
 
         say "Detect stale containers...", :magenta
-        invoke "kamal:cli:app:stale_containers", [], invoke_options
+        invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
 
         invoke "kamal:cli:app:boot", [], invoke_options
 
@@ -66,11 +75,13 @@ class Kamal::Cli::Main < Kamal::Cli::Base
 
         run_hook "pre-deploy"
 
+        push_env(invoke_options)
+
         say "Ensure app can pass healthcheck...", :magenta
         invoke "kamal:cli:healthcheck:perform", [], invoke_options
 
         say "Detect stale containers...", :magenta
-        invoke "kamal:cli:app:stale_containers", [], invoke_options
+        invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
 
         invoke "kamal:cli:app:boot", [], invoke_options
       end
@@ -92,6 +103,8 @@ class Kamal::Cli::Main < Kamal::Cli::Base
         if container_available?(version)
           run_hook "pre-deploy"
 
+          push_env(invoke_options)
+
           invoke "kamal:cli:app:boot", [], invoke_options.merge(version: version)
           rolled_back = true
         else
@@ -165,6 +178,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   end
 
   desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)"
+  option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip .env file push"
   def envify
     if destination = options[:destination]
       env_template_path = ".env.#{destination}.erb"
@@ -174,10 +188,12 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       env_path          = ".env"
     end
 
-    File.write(env_path, ERB.new(File.read(env_template_path)).result, perm: 0600)
+    File.write(env_path, ERB.new(File.read(env_template_path), trim_mode: "-").result, perm: 0600)
 
-    load_envs # reload new file
+    unless options[:skip_push]
-    invoke "kamal:cli:env:push", options
+      reload_envs
+      invoke "kamal:cli:env:push", options
+    end
   end
 
   desc "remove", "Remove Traefik, app, accessories, and registry session from servers"
@@ -252,4 +268,11 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     def deploy_options
       { "version" => KAMAL.config.version }.merge(options.without("skip_push"))
     end
+
+    def push_env(invoke_options)
+      if KAMAL.config.push_env
+        say "Pushing #{KAMAL.config.push_env} env files..."
+        invoke "kamal:cli:env:push", [], invoke_options.merge(env_type: KAMAL.config.push_env)
+      end
+    end
 end

lib/kamal/cli/prune.rb

@@ -7,7 +7,7 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
     end
   end
 
-  desc "images", "Prune dangling images"
+  desc "images", "Prune unused images"
   def images
     mutating do
       on(KAMAL.hosts) do
@@ -18,12 +18,17 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
     end
   end
 
-  desc "containers", "Prune all stopped containers, except the last 5"
+  desc "containers", "Prune all stopped containers, except the last n (default 5)"
+  option :retain, type: :numeric, default: nil, desc: "Number of containers to retain"
   def containers
+    retain = options.fetch(:retain, KAMAL.config.retain_containers)
+    raise "retain must be at least 1" if retain < 1
+
     mutating do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug
-        execute *KAMAL.prune.containers
+        execute *KAMAL.prune.app_containers(retain: retain)
+        execute *KAMAL.prune.healthcheck_containers
       end
     end
   end

lib/kamal/cli/server.rb

@@ -12,9 +12,7 @@ class Kamal::Cli::Server < Kamal::Cli::Base
           missing << host
         end
       end
-    end
 
-    on(KAMAL.hosts) do
       execute(*KAMAL.server.ensure_run_directory)
     end
 

lib/kamal/cli/templates/deploy.yml

@@ -19,6 +19,7 @@ registry:
     - KAMAL_REGISTRY_PASSWORD
 
 # Inject ENV variables into containers (secrets come from .env).
+# Remember to run `kamal env push` after making changes!
 # env:
 #   clear:
 #     DB_HOST: 192.168.0.2
@@ -52,7 +53,7 @@ registry:
 #         - MYSQL_ROOT_PASSWORD
 #     files:
 #       - config/mysql/production.cnf:/etc/mysql/my.cnf
-#       - db/production.sql.erb:/docker-entrypoint-initdb.d/setup.sql
+#       - db/production.sql:/docker-entrypoint-initdb.d/setup.sql
 #     directories:
 #       - data:/var/lib/mysql
 #   redis:
@@ -72,3 +73,29 @@ registry:
 # healthcheck:
 #   path: /healthz
 #   port: 4000
+
+# Bridge fingerprinted assets, like JS and CSS, between versions to avoid
+# hitting 404 on in-flight requests. Combines all files from new and old
+# version inside the asset_path.
+#
+# If your app is using the Sprockets gem, ensure it sets `config.assets.manifest`.
+# See https://github.com/basecamp/kamal/issues/626 for details
+#
+# asset_path: /rails/public/assets
+
+# Configure rolling deploys by setting a wait time between batches of restarts.
+# boot:
+#   limit: 10 # Can also specify as a percentage of total hosts, such as "25%"
+#   wait: 2
+
+# Configure the role used to determine the primary_host. This host takes
+# deploy locks, runs health checks during the deploy, and follow logs, etc.
+#
+# Caution: there's no support for role renaming yet, so be careful to cleanup
+#          the previous role on the deployed hosts.
+# primary_role: web
+
+# Controls if we abort when see a role with no hosts. Disabling this may be
+# useful for more complex deploy configurations.
+#
+# allow_empty_roles: false

lib/kamal/cli/templates/sample_hooks/post-traefik-reboot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooted Traefik on $KAMAL_HOSTS"

lib/kamal/cli/templates/sample_hooks/pre-build.sample

@@ -32,7 +32,7 @@ fi
 current_branch=$(git branch --show-current)
 
 if [ -z "$current_branch" ]; then
-  echo "No git remote set, aborting..." >&2
+  echo "Not on a git branch, aborting..." >&2
   exit 1
 fi
 

lib/kamal/cli/templates/sample_hooks/pre-traefik-reboot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooting Traefik on $KAMAL_HOSTS..."

lib/kamal/cli/traefik.rb

@@ -13,12 +13,18 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
   option :rolling, type: :boolean, default: false, desc: "Reboot traefik on hosts in sequence, rather than in parallel"
   def reboot
     mutating do
-      on(KAMAL.traefik_hosts, in: options[:rolling] ? :sequence : :parallel) do
+      host_groups = options[:rolling] ? KAMAL.traefik_hosts : [KAMAL.traefik_hosts]
-        execute *KAMAL.auditor.record("Rebooted traefik"), verbosity: :debug
+      host_groups.each do |hosts|
-        execute *KAMAL.registry.login
+        host_list = Array(hosts).join(",")
-        execute *KAMAL.traefik.stop
+        run_hook "pre-traefik-reboot", hosts: host_list
-        execute *KAMAL.traefik.remove_container
+        on(hosts) do
-        execute *KAMAL.traefik.run
+          execute *KAMAL.auditor.record("Rebooted traefik"), verbosity: :debug
+          execute *KAMAL.registry.login
+          execute *KAMAL.traefik.stop
+          execute *KAMAL.traefik.remove_container
+          execute *KAMAL.traefik.run
+        end
+        run_hook "post-traefik-reboot", hosts: host_list
       end
     end
   end

lib/kamal/commander.rb

@@ -24,19 +24,40 @@ class Kamal::Commander
   attr_reader :specific_roles, :specific_hosts
 
   def specific_primary!
-    self.specific_hosts = [ config.primary_web_host ]
+    self.specific_hosts = [ config.primary_host ]
   end
 
   def specific_roles=(role_names)
-    @specific_roles = config.roles.select { |r| role_names.include?(r.name) } if role_names.present?
+    if role_names.present?
+      @specific_roles = Kamal::Utils.filter_specific_items(role_names, config.roles)
+
+      if @specific_roles.empty?
+        raise ArgumentError, "No --roles match for #{role_names.join(',')}"
+      end
+
+      @specific_roles
+    end
   end
 
   def specific_hosts=(hosts)
-    @specific_hosts = config.all_hosts & hosts if hosts.present?
+    if hosts.present?
+      @specific_hosts = Kamal::Utils.filter_specific_items(hosts, config.all_hosts)
+
+      if @specific_hosts.empty?
+        raise ArgumentError, "No --hosts match for #{hosts.join(',')}"
+      end
+
+      @specific_hosts
+    end
   end
 
   def primary_host
-    specific_hosts&.first || specific_roles&.first&.primary_host || config.primary_web_host
+    # Given a list of specific roles, make an effort to match up with the primary_role
+    specific_hosts&.first || specific_roles&.detect { |role| role.name == config.primary_role }&.primary_host || specific_roles&.first&.primary_host || config.primary_host
+  end
+
+  def primary_role
+    roles_on(primary_host).first
   end
 
   def roles
@@ -51,14 +72,6 @@ class Kamal::Commander
     end
   end
 
-  def boot_strategy
-    if config.boot.limit.present?
-      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
-    else
-      {}
-    end
-  end
-
   def roles_on(host)
     roles.select { |role| role.hosts.include?(host.to_s) }.map(&:name)
   end
@@ -128,6 +141,7 @@ class Kamal::Commander
     @traefik ||= Kamal::Commands::Traefik.new(config)
   end
 
+
   def with_verbosity(level)
     old_level = self.verbosity
 
@@ -140,6 +154,14 @@ class Kamal::Commander
     SSHKit.config.output_verbosity = old_level
   end
 
+  def boot_strategy
+    if config.boot.limit.present?
+      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
+    else
+      {}
+    end
+  end
+
   def holding_lock?
     self.holding_lock
   end

lib/kamal/commands/accessory.rb

@@ -102,8 +102,8 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     make_directory accessory_config.host_env_directory
   end
 
-  def remove_env_file
+  def remove_env_files
-    [:rm, "-f", accessory_config.host_env_file_path]
+    [:rm, "-f", File.join(accessory_config.host_env_directory, "#{accessory_config.service_name}*.env")]
   end
 
   private

lib/kamal/commands/app.rb

@@ -1,4 +1,6 @@
 class Kamal::Commands::App < Kamal::Commands::Base
+  include Assets, Containers, Cord, Execution, Images, Logging
+
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 
   attr_reader :role, :role_config
@@ -16,6 +18,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
       "--name", container_name,
       *(["--hostname", hostname] if hostname),
       "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
+      "-e", "KAMAL_VERSION=\"#{config.version}\"",
       *role_config.env_args,
       *role_config.health_check_args,
       *config.logging_args,
@@ -46,51 +49,6 @@ class Kamal::Commands::App < Kamal::Commands::Base
   end
 
 
-  def logs(since: nil, lines: nil, grep: nil)
-    pipe \
-      current_running_container_id,
-      "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
-      ("grep '#{grep}'" if grep)
-  end
-
-  def follow_logs(host:, grep: nil)
-    run_over_ssh \
-      pipe(
-        current_running_container_id,
-        "xargs docker logs --timestamps --tail 10 --follow 2>&1",
-        (%(grep "#{grep}") if grep)
-      ),
-      host: host
-  end
-
-
-  def execute_in_existing_container(*command, interactive: false)
-    docker :exec,
-      ("-it" if interactive),
-      container_name,
-      *command
-  end
-
-  def execute_in_new_container(*command, interactive: false)
-    docker :run,
-      ("-it" if interactive),
-      "--rm",
-      *role_config&.env_args,
-      *config.volume_args,
-      *role_config&.option_args,
-      config.absolute_image,
-      *command
-  end
-
-  def execute_in_existing_container_over_ssh(*command, host:)
-    run_over_ssh execute_in_existing_container(*command, interactive: true), host: host
-  end
-
-  def execute_in_new_container_over_ssh(*command, host:)
-    run_over_ssh execute_in_new_container(*command, interactive: true), host: host
-  end
-
-
   def current_running_container_id
     docker :ps, "--quiet", *filter_args(statuses: ACTIVE_DOCKER_STATUSES), "--latest"
   end
@@ -109,95 +67,15 @@ class Kamal::Commands::App < Kamal::Commands::Base
       %(while read line; do echo ${line##{role_config.container_prefix}-}; done) # Extract SHA from "service-role-dest-SHA"
   end
 
-  def list_containers
-    docker :container, :ls, "--all", *filter_args
-  end
-
-  def list_container_names
-    [ *list_containers, "--format", "'{{ .Names }}'" ]
-  end
-
-  def remove_container(version:)
-    pipe \
-      container_id_for(container_name: container_name(version)),
-      xargs(docker(:container, :rm))
-  end
-
-  def rename_container(version:, new_version:)
-    docker :rename, container_name(version), container_name(new_version)
-  end
-
-  def remove_containers
-    docker :container, :prune, "--force", *filter_args
-  end
-
-  def list_images
-    docker :image, :ls, config.repository
-  end
-
-  def remove_images
-    docker :image, :prune, "--all", "--force", *filter_args
-  end
-
-  def tag_current_as_latest
-    docker :tag, config.absolute_image, config.latest_image
-  end
 
   def make_env_directory
     make_directory role_config.host_env_directory
   end
 
-  def remove_env_file
+  def remove_env_files
-    [:rm, "-f", role_config.host_env_file_path]
+    [ :rm, "-f", File.join(role_config.host_env_directory, "#{role_config.container_prefix}*.env") ]
   end
 
-  def cord(version:)
-    pipe \
-      docker(:inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", container_name(version)),
-      [:awk, "'$2 == \"#{role_config.cord_volume.container_path}\" {print $1}'"]
-  end
-
-  def tie_cord(cord)
-    create_empty_file(cord)
-  end
-
-  def cut_cord(cord)
-    remove_directory(cord)
-  end
-
-  def extract_assets
-    asset_container = "#{role_config.container_prefix}-assets"
-
-    combine \
-      make_directory(role_config.asset_extracted_path),
-      [*docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true"],
-      docker(:run, "--name", asset_container, "--detach", "--rm", config.latest_image, "sleep infinity"),
-      docker(:cp, "-L", "#{asset_container}:#{role_config.asset_path}/.", role_config.asset_extracted_path),
-      docker(:stop, "-t 1", asset_container),
-      by: "&&"
-  end
-
-  def sync_asset_volumes(old_version: nil)
-    new_extracted_path, new_volume_path = role_config.asset_extracted_path(config.version), role_config.asset_volume.host_path
-    if old_version.present?
-      old_extracted_path, old_volume_path = role_config.asset_extracted_path(old_version), role_config.asset_volume(old_version).host_path
-    end
-
-    commands = [make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path)]
-
-    if old_version.present?
-      commands << copy_contents(new_extracted_path, old_volume_path)
-      commands << copy_contents(old_extracted_path, new_volume_path)
-    end
-
-    chain *commands
-  end
-
-  def cleanup_assets
-    chain \
-      find_and_remove_older_siblings(role_config.asset_extracted_path),
-      find_and_remove_older_siblings(role_config.asset_volume_path)
-  end
 
   private
     def container_name(version = nil)
@@ -209,7 +87,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
     end
 
     def service_role_dest
-      [config.service, role, config.destination].compact.join("-")
+      [ config.service, role, config.destination ].compact.join("-")
     end
 
     def filters(statuses: nil)
@@ -221,19 +99,4 @@ class Kamal::Commands::App < Kamal::Commands::Base
         end
       end
     end
-
-    def find_and_remove_older_siblings(path)
-      [
-        :find,
-        Pathname.new(path).dirname,
-        "-maxdepth 1",
-        "-name", "'#{role_config.container_prefix}-*'",
-        "!", "-name", Pathname.new(path).basename,
-        "-exec rm -rf \"{}\" +"
-      ]
-    end
-
-    def copy_contents(source, destination)
-      [ :cp, "-rn", "#{source}/*", destination ]
-    end
 end

lib/kamal/commands/app/assets.rb

@@ -0,0 +1,51 @@
+module Kamal::Commands::App::Assets
+  def extract_assets
+    asset_container = "#{role_config.container_prefix}-assets"
+
+    combine \
+      make_directory(role_config.asset_extracted_path),
+      [*docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true"],
+      docker(:run, "--name", asset_container, "--detach", "--rm", config.latest_image, "sleep 1000000"),
+      docker(:cp, "-L", "#{asset_container}:#{role_config.asset_path}/.", role_config.asset_extracted_path),
+      docker(:stop, "-t 1", asset_container),
+      by: "&&"
+  end
+
+  def sync_asset_volumes(old_version: nil)
+    new_extracted_path, new_volume_path = role_config.asset_extracted_path(config.version), role_config.asset_volume.host_path
+    if old_version.present?
+      old_extracted_path, old_volume_path = role_config.asset_extracted_path(old_version), role_config.asset_volume(old_version).host_path
+    end
+
+    commands = [make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path)]
+
+    if old_version.present?
+      commands << copy_contents(new_extracted_path, old_volume_path, continue_on_error: true)
+      commands << copy_contents(old_extracted_path, new_volume_path, continue_on_error: true)
+    end
+
+    chain *commands
+  end
+
+  def clean_up_assets
+    chain \
+      find_and_remove_older_siblings(role_config.asset_extracted_path),
+      find_and_remove_older_siblings(role_config.asset_volume_path)
+  end
+
+  private
+    def find_and_remove_older_siblings(path)
+      [
+        :find,
+        Pathname.new(path).dirname.to_s,
+        "-maxdepth 1",
+        "-name", "'#{role_config.container_prefix}-*'",
+        "!", "-name", Pathname.new(path).basename.to_s,
+        "-exec rm -rf \"{}\" +"
+      ]
+    end
+
+    def copy_contents(source, destination, continue_on_error: false)
+      [ :cp, "-rnT", "#{source}", destination, *("|| true" if continue_on_error)]
+    end
+end

lib/kamal/commands/app/containers.rb

@@ -0,0 +1,23 @@
+module Kamal::Commands::App::Containers
+  def list_containers
+    docker :container, :ls, "--all", *filter_args
+  end
+
+  def list_container_names
+    [ *list_containers, "--format", "'{{ .Names }}'" ]
+  end
+
+  def remove_container(version:)
+    pipe \
+      container_id_for(container_name: container_name(version)),
+      xargs(docker(:container, :rm))
+  end
+
+  def rename_container(version:, new_version:)
+    docker :rename, container_name(version), container_name(new_version)
+  end
+
+  def remove_containers
+    docker :container, :prune, "--force", *filter_args
+  end
+end

lib/kamal/commands/app/cord.rb

@@ -0,0 +1,22 @@
+module Kamal::Commands::App::Cord
+  def cord(version:)
+    pipe \
+      docker(:inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", container_name(version)),
+      [:awk, "'$2 == \"#{role_config.cord_volume.container_path}\" {print $1}'"]
+  end
+
+  def tie_cord(cord)
+    create_empty_file(cord)
+  end
+
+  def cut_cord(cord)
+    remove_directory(cord)
+  end
+  
+  private    
+    def create_empty_file(file)
+      chain \
+        make_directory_for(file),
+        [:touch, file]
+    end
+end

lib/kamal/commands/app/execution.rb

@@ -0,0 +1,27 @@
+module Kamal::Commands::App::Execution
+  def execute_in_existing_container(*command, interactive: false)
+    docker :exec,
+      ("-it" if interactive),
+      container_name,
+      *command
+  end
+
+  def execute_in_new_container(*command, interactive: false)
+    docker :run,
+      ("-it" if interactive),
+      "--rm",
+      *role_config&.env_args,
+      *config.volume_args,
+      *role_config&.option_args,
+      config.absolute_image,
+      *command
+  end
+
+  def execute_in_existing_container_over_ssh(*command, host:)
+    run_over_ssh execute_in_existing_container(*command, interactive: true), host: host
+  end
+
+  def execute_in_new_container_over_ssh(*command, host:)
+    run_over_ssh execute_in_new_container(*command, interactive: true), host: host
+  end
+end

lib/kamal/commands/app/images.rb

@@ -0,0 +1,13 @@
+module Kamal::Commands::App::Images
+  def list_images
+    docker :image, :ls, config.repository
+  end
+
+  def remove_images
+    docker :image, :prune, "--all", "--force", *filter_args
+  end
+
+  def tag_current_image_as_latest
+    docker :tag, config.absolute_image, config.latest_image
+  end
+end

lib/kamal/commands/app/logging.rb

@@ -0,0 +1,18 @@
+module Kamal::Commands::App::Logging
+  def logs(since: nil, lines: nil, grep: nil)
+    pipe \
+      current_running_container_id,
+      "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
+      ("grep '#{grep}'" if grep)
+  end
+
+  def follow_logs(host:, grep: nil)
+    run_over_ssh \
+      pipe(
+        current_running_container_id,
+        "xargs docker logs --timestamps --tail 10 --follow 2>&1",
+        (%(grep "#{grep}") if grep)
+      ),
+      host: host
+  end
+end

lib/kamal/commands/base.rb

@@ -18,7 +18,7 @@ module Kamal::Commands
         elsif config.ssh.proxy && config.ssh.proxy.is_a?(Net::SSH::Proxy::Command)
           cmd << " -o ProxyCommand='#{config.ssh.proxy.command_line_template}'"
         end
-        cmd << " -t #{config.ssh.user}@#{host} '#{command.join(" ")}'"
+        cmd << " -t #{config.ssh.user}@#{host} -p #{config.ssh.port} '#{command.join(" ")}'"
       end
     end
 
@@ -73,11 +73,5 @@ module Kamal::Commands
       def tags(**details)
         Kamal::Tags.from_config(config, **details)
       end
-
-      def create_empty_file(file)
-        chain \
-          make_directory_for(file),
-          [:touch, file]
-      end
   end
 end

lib/kamal/commands/builder.rb

@@ -1,7 +1,7 @@
 require "active_support/core_ext/string/filters"
 
 class Kamal::Commands::Builder < Kamal::Commands::Base
-  delegate :create, :remove, :push, :clean, :pull, :info, to: :target
+  delegate :create, :remove, :push, :clean, :pull, :info, :validate_image, to: :target
 
   def name
     target.class.to_s.remove("Kamal::Commands::Builder::").underscore.inquiry

lib/kamal/commands/builder/base.rb

@@ -3,7 +3,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
   class BuilderError < StandardError; end
 
   delegate :argumentize, to: Kamal::Utils
-  delegate :args, :secrets, :dockerfile, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, to: :builder_config
+  delegate :args, :secrets, :dockerfile, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, :ssh, to: :builder_config
 
   def clean
     docker :image, :rm, "--force", config.absolute_image
@@ -14,13 +14,19 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
   end
 
   def build_options
-    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile ]
+    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_ssh ]
   end
 
   def build_context
     config.builder.context
   end
 
+  def validate_image
+    pipe \
+      docker(:inspect, "-f", "'{{ .Config.Labels.service }}'", config.absolute_image),
+      [:grep, "-x", config.service, "||", "(echo \"Image #{config.absolute_image} is missing the `service` label\" && exit 1)"]
+  end
+
 
   private
     def build_tags
@@ -54,6 +60,10 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
       end
     end
 
+    def build_ssh
+      argumentize "--ssh", ssh if ssh.present?
+    end
+
     def builder_config
       config.builder
     end

lib/kamal/commands/builder/multiarch.rb

@@ -10,7 +10,7 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
   def push
     docker :buildx, :build,
       "--push",
-      "--platform", "linux/amd64,linux/arm64",
+      "--platform", platform_names,
       "--builder", builder_name,
       *build_options,
       build_context
@@ -26,4 +26,12 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
     def builder_name
       "kamal-#{config.service}-multiarch"
     end
+
+    def platform_names
+      if local_arch
+        "linux/#{local_arch}"
+      else
+        "linux/amd64,linux/arm64"
+      end
+    end
 end

lib/kamal/commands/docker.rb

@@ -16,6 +16,6 @@ class Kamal::Commands::Docker < Kamal::Commands::Base
 
   # Do we have superuser access to install Docker and start system services?
   def superuser?
-    [ '[ "${EUID:-$(id -u)}" -eq 0 ]' ]
+    [ '[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null' ]
   end
 end

lib/kamal/commands/healthcheck.rb

@@ -1,20 +1,20 @@
 class Kamal::Commands::Healthcheck < Kamal::Commands::Base
 
   def run
-    web = config.role(:web)
+    primary = config.role(config.primary_role)
 
     docker :run,
       "--detach",
       "--name", container_name_with_version,
       "--publish", "#{exposed_port}:#{config.healthcheck["port"]}",
-      "--label", "service=#{container_name}",
+      "--label", "service=#{config.healthcheck_service}",
-      "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
+      "-e", "KAMAL_CONTAINER_NAME=\"#{config.healthcheck_service}\"",
-      *web.env_args,
+      *primary.env_args,
-      *web.health_check_args(cord: false),
+      *primary.health_check_args(cord: false),
       *config.volume_args,
-      *web.option_args,
+      *primary.option_args,
       config.absolute_image,
-      web.cmd
+      primary.cmd
   end
 
   def status
@@ -26,7 +26,7 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
   end
 
   def logs
-    pipe container_id, xargs(docker(:logs, "--tail", 50, "2>&1"))
+    pipe container_id, xargs(docker(:logs, "--tail", log_lines, "2>&1"))
   end
 
   def stop
@@ -38,12 +38,8 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
   end
 
   private
-    def container_name
-      [ "healthcheck", config.service, config.destination ].compact.join("-")
-    end
-
     def container_name_with_version
-      "#{container_name}-#{config.version}"
+      "#{config.healthcheck_service}-#{config.version}"
     end
 
     def container_id
@@ -57,4 +53,8 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
     def exposed_port
       config.healthcheck["exposed_port"]
     end
+
+    def log_lines
+      config.healthcheck["log_lines"]
+    end
 end

lib/kamal/commands/lock.rb

@@ -1,5 +1,6 @@
 require "active_support/duration"
 require "time"
+require "base64"
 
 class Kamal::Commands::Lock < Kamal::Commands::Base
   def acquire(message, version)
@@ -56,7 +57,7 @@ class Kamal::Commands::Lock < Kamal::Commands::Base
     end
 
     def locked_by
-      `git config user.name`.strip
+      Kamal::Git.user_name
     rescue Errno::ENOENT
       "Unknown"
     end

lib/kamal/commands/prune.rb

@@ -3,7 +3,7 @@ require "active_support/core_ext/numeric/time"
 
 class Kamal::Commands::Prune < Kamal::Commands::Base
   def dangling_images
-    docker :image, :prune, "--force", "--filter", "label=service=#{config.service}", "--filter", "dangling=true"
+    docker :image, :prune, "--force", "--filter", "label=service=#{config.service}"
   end
 
   def tagged_images
@@ -13,13 +13,17 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
       "while read image tag; do docker rmi $tag; done"
   end
 
-  def containers(keep_last: 5)
+  def app_containers(retain:)
     pipe \
       docker(:ps, "-q", "-a", *service_filter, *stopped_containers_filters),
-      "tail -n +#{keep_last + 1}",
+      "tail -n +#{retain + 1}",
       "while read container_id; do docker rm $container_id; done"
   end
 
+  def healthcheck_containers
+    docker :container, :prune, "--force", *healthcheck_service_filter
+  end
+
   private
     def stopped_containers_filters
       [ "created", "exited", "dead" ].flat_map { |status| ["--filter", "status=#{status}"] }
@@ -35,4 +39,8 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
     def service_filter
       [ "--filter", "label=service=#{config.service}" ]
     end
-end
+
+    def healthcheck_service_filter
+      [ "--filter", "label=service=#{config.healthcheck_service}" ]
+    end
+  end

lib/kamal/commands/traefik.rb

@@ -1,11 +1,19 @@
 class Kamal::Commands::Traefik < Kamal::Commands::Base
-  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
-  DEFAULT_IMAGE = "traefik:v2.9"
+  DEFAULT_IMAGE = "traefik:v2.10"
   CONTAINER_PORT = 80
   DEFAULT_ARGS = {
     'log.level' => 'DEBUG'
   }
+  DEFAULT_LABELS = {
+    # These ensure we serve a 502 rather than a 404 if no containers are available
+    "traefik.http.routers.catchall.entryPoints" => "http",
+    "traefik.http.routers.catchall.rule" => "PathPrefix(`/`)",
+    "traefik.http.routers.catchall.service" => "unavailable",
+    "traefik.http.routers.catchall.priority" => 1,
+    "traefik.http.services.unavailable.loadbalancer.server.port" => "0"
+  }
 
   def run
     docker :run, "--name traefik",
@@ -64,19 +72,23 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
   end
 
   def env_file
-    env_file_with_secrets config.traefik.fetch("env", {})
+    Kamal::EnvFiles.new(config.traefik.fetch("env", {}))
   end
 
-  def host_env_file_path
+  def host_clear_env_file_path
-    File.join host_env_directory, "traefik.env"
+    File.join host_env_directory, "traefik-clear.env"
+  end
+
+  def host_secret_env_file_path
+    File.join host_env_directory, "traefik-secret.env"
   end
 
   def make_env_directory
     make_directory(host_env_directory)
   end
 
-  def remove_env_file
+  def remove_env_files
-    [:rm, "-f", host_env_file_path]
+    [:rm, "-f", File.join(host_env_directory, "traefik*.env")]
   end
 
   private
@@ -89,7 +101,10 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
     end
 
     def env_args
-      argumentize "--env-file", host_env_file_path
+      [
+        *argumentize("--env-file", host_secret_env_file_path),
+        *argumentize("--env-file", host_clear_env_file_path)
+      ]
     end
 
     def host_env_directory
@@ -97,7 +112,7 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
     end
 
     def labels
-      config.traefik["labels"] || []
+      DEFAULT_LABELS.merge(config.traefik["labels"] || {})
     end
 
     def image

lib/kamal/configuration.rb

@@ -6,11 +6,10 @@ require "erb"
 require "net/ssh/proxy/jump"
 
 class Kamal::Configuration
-  delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
+  delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, :push_env, to: :raw_config, allow_nil: true
   delegate :argumentize, :optionize, to: Kamal::Utils
 
-  attr_accessor :destination
+  attr_reader :destination, :raw_config
-  attr_accessor :raw_config
 
   class << self
     def create_from(config_file:, destination: nil, version: nil)
@@ -26,7 +25,9 @@ class Kamal::Configuration
 
       def load_config_file(file)
         if file.exist?
-          YAML.load(ERB.new(IO.read(file)).result).symbolize_keys
+          # Newer Psych doesn't load aliases by default
+          load_method = YAML.respond_to?(:unsafe_load) ? :unsafe_load : :load
+          YAML.send(load_method, ERB.new(IO.read(file)).result).symbolize_keys
         else
           raise "Configuration file not found in #{file}"
         end
@@ -54,21 +55,20 @@ class Kamal::Configuration
   end
 
   def abbreviated_version
-    Kamal::Utils.abbreviate_version(version)
+    if version
-  end
+      # Don't abbreviate <sha>_uncommitted_<etc>
-
+      if version.include?("_")
-  def run_directory
+        version
-    raw_config.run_directory || ".kamal"
+      else
-  end
+        version[0...7]
-
+      end
-  def run_directory_as_docker_volume
-    if Pathname.new(run_directory).absolute?
-      run_directory
-    else
-      File.join "$(pwd)", run_directory
     end
   end
 
+  def minimum_version
+    raw_config.minimum_version
+  end
+
 
   def roles
     @roles ||= role_names.collect { |role_name| Role.new(role_name, config: self) }
@@ -91,19 +91,22 @@ class Kamal::Configuration
     roles.flat_map(&:hosts).uniq
   end
 
-  def primary_web_host
+  def primary_host
-    role(:web).primary_host
+    role(primary_role)&.primary_host
+  end
+
+  def traefik_roles
+    roles.select(&:running_traefik?)
+  end
+
+  def traefik_role_names
+    traefik_roles.flat_map(&:name)
   end
 
   def traefik_hosts
-    roles.select(&:running_traefik?).flat_map(&:hosts).uniq
+    traefik_roles.flat_map(&:hosts).uniq
   end
 
-  def boot
-    Kamal::Configuration::Boot.new(config: self)
-  end
-
-
   def repository
     [ raw_config.registry["server"], image ].compact.join("/")
   end
@@ -120,6 +123,14 @@ class Kamal::Configuration
     "#{service}-#{version}"
   end
 
+  def require_destination?
+    raw_config.require_destination
+  end
+
+  def retain_containers
+    raw_config.retain_containers || 5
+  end
+
 
   def volume_args
     if raw_config.volumes.present?
@@ -139,6 +150,18 @@ class Kamal::Configuration
   end
 
 
+  def boot
+    Kamal::Configuration::Boot.new(config: self)
+  end
+
+  def builder
+    Kamal::Configuration::Builder.new(config: self)
+  end
+
+  def traefik
+    raw_config.traefik || {}
+  end
+
   def ssh
     Kamal::Configuration::Ssh.new(config: self)
   end
@@ -149,27 +172,69 @@ class Kamal::Configuration
 
 
   def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }.merge(raw_config.healthcheck || {})
+    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }.merge(raw_config.healthcheck || {})
+  end
+
+  def healthcheck_service
+    [ "healthcheck", service, destination ].compact.join("-")
   end
 
   def readiness_delay
     raw_config.readiness_delay || 7
   end
 
-  def minimum_version
+  def run_id
-    raw_config.minimum_version
+    @run_id ||= SecureRandom.hex(16)
   end
 
+
+  def run_directory
+    raw_config.run_directory || ".kamal"
+  end
+
+  def run_directory_as_docker_volume
+    if Pathname.new(run_directory).absolute?
+      run_directory
+    else
+      File.join "$(pwd)", run_directory
+    end
+  end
+
+  def hooks_path
+    raw_config.hooks_path || ".kamal/hooks"
+  end
+
+  def host_env_directory
+    "#{run_directory}/env"
+  end
+
+  def asset_path
+    raw_config.asset_path
+  end
+
+  def primary_role
+    raw_config.primary_role || "web"
+  end
+
+  def allow_empty_roles?
+    raw_config.allow_empty_roles
+  end
+
+
   def valid?
-    ensure_required_keys_present && ensure_valid_kamal_version
+    ensure_destination_if_required \
+      && ensure_required_keys_present \
+      && ensure_valid_kamal_version \
+      && ensure_retain_containers_valid \
+      && ensure_valid_service_name \
+      && ensure_push_env_valid
   end
 
-
   def to_h
     {
       roles: role_names,
       hosts: all_hosts,
-      primary_host: primary_web_host,
+      primary_host: primary_host,
       version: version,
       repository: repository,
       absolute_image: absolute_image,
@@ -184,39 +249,17 @@ class Kamal::Configuration
     }.compact
   end
 
-  def traefik
-    raw_config.traefik || {}
-  end
-
-  def hooks_path
-    raw_config.hooks_path || ".kamal/hooks"
-  end
-
-  def builder
-    Kamal::Configuration::Builder.new(config: self)
-  end
-
-  # Will raise KeyError if any secret ENVs are missing
-  def ensure_env_available
-    roles.each(&:env_file)
-
-    true
-  end
-
-  def host_env_directory
-    "#{run_directory}/env"
-  end
-
-  def run_id
-    @run_id ||= SecureRandom.hex(16)
-  end
-
-  def asset_path
-    raw_config.asset_path
-  end
 
   private
     # Will raise ArgumentError if any required config keys are missing
+    def ensure_destination_if_required
+      if require_destination? && destination.nil?
+        raise ArgumentError, "You must specify a destination"
+      end
+
+      true
+    end
+
     def ensure_required_keys_present
       %i[ service image registry servers ].each do |key|
         raise ArgumentError, "Missing required configuration for #{key}" unless raw_config[key].present?
@@ -230,15 +273,31 @@ class Kamal::Configuration
         raise ArgumentError, "You must specify a password for the registry in config/deploy.yml (or set the ENV variable if that's used)"
       end
 
-      roles.each do |role|
+      unless role_names.include?(primary_role)
-        if role.hosts.empty?
+        raise ArgumentError, "The primary_role #{primary_role} isn't defined"
-          raise ArgumentError, "No servers specified for the #{role.name} role"
+      end
+
+      if role(primary_role).hosts.empty?
+        raise ArgumentError, "No servers specified for the #{primary_role} primary_role"
+      end
+
+      unless allow_empty_roles?
+        roles.each do |role|
+          if role.hosts.empty?
+            raise ArgumentError, "No servers specified for the #{role.name} role. You can ignore this with allow_empty_roles: true"
+          end
         end
       end
 
       true
     end
 
+    def ensure_valid_service_name
+      raise ArgumentError, "Service name can only include alphanumeric characters, hyphens, and underscores" unless raw_config[:service] =~ /^[a-z0-9-_]+$/
+
+      true
+    end
+
     def ensure_valid_kamal_version
       if minimum_version && Gem::Version.new(minimum_version) > Gem::Version.new(Kamal::VERSION)
         raise ArgumentError, "Current version is #{Kamal::VERSION}, minimum required is #{minimum_version}"
@@ -247,6 +306,20 @@ class Kamal::Configuration
       true
     end
 
+    def ensure_retain_containers_valid
+      raise ArgumentError, "Must retain at least 1 container" if retain_containers < 1
+
+      true
+    end
+
+    def ensure_push_env_valid
+      if raw_config.push_env && !%w[ all clear secret ].include?(raw_config.push_env)
+        raise ArgumentError, "push_env must be one of `all`, `clear` `secret`"
+      end
+
+      true
+    end
+
 
     def role_names
       raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort
@@ -254,10 +327,8 @@ class Kamal::Configuration
 
     def git_version
       @git_version ||=
-        if system("git rev-parse")
+        if Kamal::Git.used?
-          uncommitted_suffix = Kamal::Utils.uncommitted_changes.present? ? "_uncommitted_#{SecureRandom.hex(8)}" : ""
+          [ Kamal::Git.revision, Kamal::Git.uncommitted_changes.present? ? "_uncommitted_#{SecureRandom.hex(8)}" : "" ].join
-
-          "#{`git rev-parse HEAD`.strip}#{uncommitted_suffix}"
         else
           raise "Can't use commit hash as version, no git repository found in #{Dir.pwd}"
         end

lib/kamal/configuration/accessory.rb

@@ -1,5 +1,5 @@
 class Kamal::Configuration::Accessory
-  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_accessor :name, :specifics
 
@@ -8,7 +8,7 @@ class Kamal::Configuration::Accessory
   end
 
   def service_name
-    "#{config.service}-#{name}"
+    specifics["service"] || "#{config.service}-#{name}"
   end
 
   def image
@@ -46,19 +46,26 @@ class Kamal::Configuration::Accessory
   end
 
   def env_file
-    env_file_with_secrets env
+    Kamal::EnvFiles.new(env)
   end
 
   def host_env_directory
     File.join config.host_env_directory, "accessories"
   end
 
-  def host_env_file_path
+  def host_clear_env_file_path
-    File.join host_env_directory, "#{service_name}.env"
+    File.join host_env_directory, "#{service_name}-clear.env"
+  end
+
+  def host_secret_env_file_path
+    File.join host_env_directory, "#{service_name}-secret.env"
   end
 
   def env_args
-    argumentize "--env-file", host_env_file_path
+    [
+      *argumentize("--env-file", host_secret_env_file_path),
+      *argumentize("--env-file", host_clear_env_file_path)
+    ]
   end
 
   def files
@@ -70,8 +77,8 @@ class Kamal::Configuration::Accessory
 
   def directories
     specifics["directories"]&.to_h do |host_to_container_mapping|
-      host_relative_path, container_path = host_to_container_mapping.split(":")
+      host_path, container_path = host_to_container_mapping.split(":")
-      [ expand_host_path(host_relative_path), container_path ]
+      [ expand_host_path(host_path), container_path ]
     end || {}
   end
 
@@ -138,13 +145,17 @@ class Kamal::Configuration::Accessory
 
     def remote_directories_as_volumes
       specifics["directories"]&.collect do |host_to_container_mapping|
-        host_relative_path, container_path = host_to_container_mapping.split(":")
+        host_path, container_path = host_to_container_mapping.split(":")
-        [ expand_host_path(host_relative_path), container_path ].join(":")
+        [ expand_host_path(host_path), container_path ].join(":")
       end || []
     end
 
-    def expand_host_path(host_relative_path)
+    def expand_host_path(host_path)
-      "#{service_data_directory}/#{host_relative_path}"
+      absolute_path?(host_path) ? host_path : "#{service_data_directory}/#{host_path}"
+    end
+
+    def absolute_path?(path)
+      Pathname.new(path).absolute?
     end
 
     def service_data_directory

lib/kamal/configuration/builder.rb

@@ -81,6 +81,10 @@ class Kamal::Configuration::Builder
     end
   end
 
+  def ssh
+    @options["ssh"]
+  end
+
   private
     def valid?
       if @options["cache"] && @options["cache"]["type"]

lib/kamal/configuration/role.rb

@@ -1,6 +1,6 @@
 class Kamal::Configuration::Role
   CORD_FILE = "cord"
-  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_accessor :name
 
@@ -16,6 +16,18 @@ class Kamal::Configuration::Role
     @hosts ||= extract_hosts_from_config
   end
 
+  def cmd
+    specializations["cmd"]
+  end
+
+  def option_args
+    if args = specializations["options"]
+      optionize args
+    else
+      []
+    end
+  end
+
   def labels
     default_labels.merge(traefik_labels).merge(custom_labels)
   end
@@ -24,6 +36,7 @@ class Kamal::Configuration::Role
     argumentize "--label", labels
   end
 
+
   def env
     if config.env && config.env["secret"]
       merged_env_with_secrets
@@ -33,25 +46,33 @@ class Kamal::Configuration::Role
   end
 
   def env_file
-    env_file_with_secrets env
+    Kamal::EnvFiles.new(env)
   end
 
   def host_env_directory
     File.join config.host_env_directory, "roles"
   end
 
-  def host_env_file_path
+  def host_clear_env_file_path
-    File.join host_env_directory, "#{[config.service, name, config.destination].compact.join("-")}.env"
+    host_env_file_path(:clear)
+  end
+
+  def host_secret_env_file_path
+    host_env_file_path(:secret)
   end
 
   def env_args
-    argumentize "--env-file", host_env_file_path
+    [
+      *argumentize("--env-file", host_secret_env_file_path),
+      *argumentize("--env-file", host_clear_env_file_path)
+    ]
   end
 
   def asset_volume_args
     asset_volume&.docker_args
   end
 
+
   def health_check_args(cord: true)
     if health_check_cmd.present?
       if cord && uses_cord?
@@ -77,6 +98,20 @@ class Kamal::Configuration::Role
     health_check_options["interval"] || "1s"
   end
 
+
+  def running_traefik?
+    if specializations["traefik"].nil?
+      primary?
+    else
+      specializations["traefik"]
+    end
+  end
+
+  def primary?
+    @config.primary_role == name
+  end
+
+
   def uses_cord?
     running_traefik? && cord_volume && health_check_cmd.present?
   end
@@ -106,22 +141,6 @@ class Kamal::Configuration::Role
   end
 
 
-  def cmd
-    specializations["cmd"]
-  end
-
-  def option_args
-    if args = specializations["options"]
-      optionize args
-    else
-      []
-    end
-  end
-
-  def running_traefik?
-    name.web? || specializations["traefik"]
-  end
-
   def container_name(version = nil)
     [ container_prefix, version || config.version ].compact.join("-")
   end
@@ -130,6 +149,7 @@ class Kamal::Configuration::Role
     [ config.service, name, config.destination ].compact.join("-")
   end
 
+
   def asset_path
     specializations["asset_path"] || config.asset_path
   end
@@ -180,6 +200,7 @@ class Kamal::Configuration::Role
           "traefik.http.services.#{traefik_service}.loadbalancer.server.scheme" => "http",
 
           "traefik.http.routers.#{traefik_service}.rule" => "PathPrefix(`/`)",
+          "traefik.http.routers.#{traefik_service}.priority" => "2",
           "traefik.http.middlewares.#{traefik_service}-retry.retry.attempts" => "5",
           "traefik.http.middlewares.#{traefik_service}-retry.retry.initialinterval" => "500ms",
           "traefik.http.routers.#{traefik_service}.middlewares" => "#{traefik_service}-retry@docker"
@@ -225,10 +246,14 @@ class Kamal::Configuration::Role
         clear_app_env  = config.env["secret"] ? Array(config.env["clear"]) : Array(config.env["clear"] || config.env)
         clear_role_env = specialized_env["secret"] ? Array(specialized_env["clear"]) : Array(specialized_env["clear"] || specialized_env)
 
-        new_env["clear"] = (clear_app_env + clear_role_env).uniq
+        new_env["clear"] = clear_app_env.to_h.merge(clear_role_env.to_h)
       end
     end
 
+    def host_env_file_path(env_type)
+      File.join host_env_directory, "#{[container_prefix, env_type].compact.join("-")}.env"
+    end
+
     def http_health_check(port:, path:)
       "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present?
     end

lib/kamal/configuration/ssh.rb

@@ -9,6 +9,10 @@ class Kamal::Configuration::Ssh
     config.fetch("user", "root")
   end
 
+  def port
+    config.fetch("port", 22)
+  end
+
   def proxy
     if (proxy = config["proxy"])
       Net::SSH::Proxy::Jump.new(proxy.include?("@") ? proxy : "root@#{proxy}")
@@ -18,7 +22,7 @@ class Kamal::Configuration::Ssh
   end
 
   def options
-    { user: user, proxy: proxy, auth_methods: [ "publickey" ], logger: logger, keepalive: true, keepalive_interval: 30 }.compact
+    { user: user, port: port, proxy: proxy, logger: logger, keepalive: true, keepalive_interval: 30 }.compact
   end
 
   def to_h

lib/kamal/env_files.rb

@@ -0,0 +1,44 @@
+# Encode an env hash as a string where secret values have been looked up and all values escaped for Docker.
+class Kamal::EnvFiles
+  def initialize(env)
+    @env = env
+  end
+
+  def secret
+    env_file do
+      @env["secret"]&.to_h { |key| [ key, ENV.fetch(key) ] }
+    end
+  end
+
+  def clear
+    env_file do
+      if (secrets = @env["secret"]).present?
+        @env["clear"]
+      else
+        @env.fetch("clear", @env)
+      end
+    end
+  end
+
+  private
+    def docker_env_file_line(key, value)
+      "#{key.to_s}=#{escape_docker_env_file_value(value)}\n"
+    end
+
+    # Escape a value to make it safe to dump in a docker file.
+    def escape_docker_env_file_value(value)
+      # Doublequotes are treated literally in docker env files
+      # so remove leading and trailing ones and unescape any others
+      value.to_s.dump[1..-2].gsub(/\\"/, "\"")
+    end
+
+    def env_file(&block)
+      StringIO.new.tap do |contents|
+        block.call&.each do |key, value|
+          contents << docker_env_file_line(key, value)
+        end
+        # Ensure the file has some contents to avoid the SSHKit empty file warning
+        contents << "\n" if contents.length == 0
+      end.string
+    end
+end

lib/kamal/git.rb

@@ -0,0 +1,19 @@
+module Kamal::Git
+  extend self
+
+  def used?
+    system("git rev-parse")
+  end
+
+  def user_name
+    `git config user.name`.strip
+  end
+
+  def revision
+    `git rev-parse HEAD`.strip
+  end
+
+  def uncommitted_changes
+    `git status --porcelain`.strip
+  end
+end

lib/kamal/sshkit_with_ext.rb

@@ -1,5 +1,6 @@
 require "sshkit"
 require "sshkit/dsl"
+require "net/scp"
 require "active_support/core_ext/hash/deep_merge"
 require "json"
 

lib/kamal/utils.rb

@@ -16,26 +16,6 @@ module Kamal::Utils
     end
   end
 
-  def env_file_with_secrets(env)
-    env_file = StringIO.new.tap do |contents|
-      if (secrets = env["secret"]).present?
-        env.fetch("secret", env)&.each do |key|
-          contents << docker_env_file_line(key, ENV.fetch(key))
-        end
-        env["clear"]&.each do |key, value|
-          contents << docker_env_file_line(key, value)
-        end
-      else
-        env.fetch("clear", env)&.each do |key, value|
-          contents << docker_env_file_line(key, value)
-        end
-      end
-    end.string
-
-    # Ensure the file has some contents to avoid the SSHKIT empty file warning
-    env_file || "\n"
-  end
-
   # Returns a list of shell-dashed option arguments. If the value is true, it's treated like a value-less option.
   def optionize(args, with: nil)
     options = if with
@@ -72,19 +52,6 @@ module Kamal::Utils
     end
   end
 
-  def unredacted(value)
-    case
-    when value.respond_to?(:unredacted)
-      value.unredacted
-    when value.respond_to?(:transform_values)
-      value.transform_values { |value| unredacted value }
-    when value.respond_to?(:map)
-      value.map { |element| unredacted element }
-    else
-      value
-    end
-  end
-
   # Escape a value to make it safe for shell use.
   def escape_shell_value(value)
     value.to_s.dump
@@ -92,27 +59,19 @@ module Kamal::Utils
       .gsub(DOLLAR_SIGN_WITHOUT_SHELL_EXPANSION_REGEX, '\$')
   end
 
-  # Abbreviate a git revhash for concise display
+  # Apply a list of host or role filters, including wildcard matches
-  def abbreviate_version(version)
+  def filter_specific_items(filters, items)
-    if version
+    matches = []
-      # Don't abbreviate <sha>_uncommitted_<etc>
+
-      if version.include?("_")
+    Array(filters).select do |filter|
-        version
+      matches += Array(items).select do |item|
-      else
+        # Only allow * for a wildcard
-        version[0...7]
+        pattern = Regexp.escape(filter).gsub('\*', '.*')
+        # items are roles or hosts
+        (item.respond_to?(:name) ? item.name : item).match(/^#{pattern}$/)
       end
     end
-  end
 
-  def uncommitted_changes
+    matches
-    `git status --porcelain`.strip
-  end
-
-  def docker_env_file_line(key, value)
-    if key.include?("\n") || value.to_s.include?("\n")
-      raise ArgumentError, "docker env file format does not support newlines in keys or values, key: #{key}"
-    end
-
-    "#{key.to_s}=#{value.to_s}\n"
   end
 end

lib/kamal/utils/healthcheck_poller.rb

@@ -1,64 +0,0 @@
-class Kamal::Utils::HealthcheckPoller
-  TRAEFIK_UPDATE_DELAY = 2
-
-  class HealthcheckError < StandardError; end
-
-  class << self
-    def wait_for_healthy(pause_after_ready: false, &block)
-      attempt = 1
-      max_attempts = KAMAL.config.healthcheck["max_attempts"]
-
-      begin
-        case status = block.call
-        when "healthy"
-          sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
-        when "running" # No health check configured
-          sleep KAMAL.config.readiness_delay if pause_after_ready
-        else
-          raise HealthcheckError, "container not ready (#{status})"
-        end
-      rescue HealthcheckError => e
-        if attempt <= max_attempts
-          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-          sleep attempt
-          attempt += 1
-          retry
-        else
-          raise
-        end
-      end
-
-      info "Container is healthy!"
-    end
-
-    def wait_for_unhealthy(pause_after_ready: false, &block)
-      attempt = 1
-      max_attempts = KAMAL.config.healthcheck["max_attempts"]
-
-      begin
-        case status = block.call
-        when "unhealthy"
-          sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
-        else
-          raise HealthcheckError, "container not unhealthy (#{status})"
-        end
-      rescue HealthcheckError => e
-        if attempt <= max_attempts
-          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-          sleep attempt
-          attempt += 1
-          retry
-        else
-          raise
-        end
-      end
-
-      info "Container is unhealthy!"
-    end
-
-    private
-      def info(message)
-        SSHKit.config.output.info(message)
-      end
-  end
-end

lib/kamal/utils/sensitive.rb

@@ -1,4 +1,5 @@
 require "active_support/core_ext/module/delegation"
+require "sshkit"
 
 class Kamal::Utils::Sensitive
   # So SSHKit knows to redact these values.

lib/kamal/version.rb

@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "0.16.1"
+  VERSION = "1.3.1"
 end

test/cli/accessory_test.rb

@@ -7,7 +7,7 @@ class CliAccessoryTest < CliTestCase
 
     run_command("boot", "mysql").tap do |output|
       assert_match /docker login.*on 1.1.1.3/, output
-      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql-secret.env --env-file .kamal/env/accessories/app-mysql-clear.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
     end
   end
 
@@ -21,9 +21,9 @@ class CliAccessoryTest < CliTestCase
       assert_match /docker login.*on 1.1.1.3/, output
       assert_match /docker login.*on 1.1.1.1/, output
       assert_match /docker login.*on 1.1.1.2/, output
-      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql-secret.env --env-file .kamal/env/accessories/app-mysql-clear.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
-      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis-secret.env --env-file .kamal/env/accessories/app-redis-clear.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
-      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis-secret.env --env-file .kamal/env/accessories/app-redis-clear.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
     end
   end
 
@@ -48,6 +48,18 @@ class CliAccessoryTest < CliTestCase
     run_command("reboot", "mysql")
   end
 
+  test "reboot all" do
+    Kamal::Commands::Registry.any_instance.expects(:login).times(3)
+    Kamal::Cli::Accessory.any_instance.expects(:stop).with("mysql")
+    Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("mysql")
+    Kamal::Cli::Accessory.any_instance.expects(:boot).with("mysql", login: false)
+    Kamal::Cli::Accessory.any_instance.expects(:stop).with("redis")
+    Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("redis")
+    Kamal::Cli::Accessory.any_instance.expects(:boot).with("redis", login: false)
+
+    run_command("reboot", "all")
+  end
+
   test "start" do
     assert_match "docker container start app-mysql", run_command("start", "mysql")
   end
@@ -97,7 +109,7 @@ class CliAccessoryTest < CliTestCase
 
   test "logs with follow" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.3 'docker logs app-mysql --timestamps --tail 10 --follow 2>&1'")
+      .with("ssh -t root@1.1.1.3 -p 22 'docker logs app-mysql --timestamps --tail 10 --follow 2>&1'")
 
     assert_match "docker logs app-mysql --timestamps --tail 10 --follow 2>&1", run_command("logs", "mysql", "--follow")
   end
@@ -136,6 +148,30 @@ class CliAccessoryTest < CliTestCase
     assert_match "rm -rf app-mysql", run_command("remove_service_directory", "mysql")
   end
 
+  test "hosts param respected" do
+    Kamal::Cli::Accessory.any_instance.expects(:directories).with("redis")
+    Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis")
+
+    run_command("boot", "redis", "--hosts", "1.1.1.1").tap do |output|
+      assert_match /docker login.*on 1.1.1.1/, output
+      refute_match /docker login.*on 1.1.1.2/, output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis-secret.env --env-file .kamal/env/accessories/app-redis-clear.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
+      refute_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis-secret.env --env-file .kamal/env/accessories/app-redis-clear.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
+    end
+  end
+
+  test "hosts param intersected with configuration" do
+    Kamal::Cli::Accessory.any_instance.expects(:directories).with("redis")
+    Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis")
+
+    run_command("boot", "redis", "--hosts", "1.1.1.1,1.1.1.3").tap do |output|
+      assert_match /docker login.*on 1.1.1.1/, output
+      refute_match /docker login.*on 1.1.1.3/, output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis-secret.env --env-file .kamal/env/accessories/app-redis-clear.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
+      refute_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis-secret.env --env-file .kamal/env/accessories/app-redis-clear.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.3", output
+    end
+  end
+
   private
     def run_command(*command)
       stdouted { Kamal::Cli::Accessory.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }

test/cli/app_test.rb

@@ -27,7 +27,7 @@ class CliAppTest < CliTestCase
       .returns("123") # old version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
       .returns("cordfile") # old version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -55,8 +55,6 @@ class CliAppTest < CliTestCase
   end
 
   test "boot errors leave lock in place" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
-
     Kamal::Cli::App.any_instance.expects(:using_version).raises(RuntimeError)
 
     assert !KAMAL.holding_lock?
@@ -66,6 +64,34 @@ class CliAppTest < CliTestCase
     assert KAMAL.holding_lock?
   end
 
+  test "boot with assets" do
+    Object.any_instance.stubs(:sleep)
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678") # running version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("123").twice # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .returns("") # old version
+
+    run_command("boot", config: :with_assets).tap do |output|
+      assert_match "docker tag dhh/app:latest dhh/app:latest", output
+      assert_match "/usr/bin/env mkdir -p .kamal/assets/volumes/app-web-latest ; cp -rnT .kamal/assets/extracted/app-web-latest .kamal/assets/volumes/app-web-latest ; cp -rnT .kamal/assets/extracted/app-web-latest .kamal/assets/volumes/app-web-123 || true ; cp -rnT .kamal/assets/extracted/app-web-123 .kamal/assets/volumes/app-web-latest || true", output
+      assert_match "/usr/bin/env mkdir -p .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets 2> /dev/null || true && docker run --name app-web-assets --detach --rm dhh/app:latest sleep 1000000 && docker cp -L app-web-assets:/public/assets/. .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets", output
+      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} /, output
+      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+      assert_match "/usr/bin/env find .kamal/assets/extracted -maxdepth 1 -name 'app-web-*' ! -name app-web-latest -exec rm -rf \"{}\" + ; find .kamal/assets/volumes -maxdepth 1 -name 'app-web-*' ! -name app-web-latest -exec rm -rf \"{}\" +", output
+    end
+  end
+
   test "start" do
     run_command("start").tap do |output|
       assert_match "docker start app-web-999", output
@@ -133,7 +159,7 @@ class CliAppTest < CliTestCase
 
   test "exec" do
     run_command("exec", "ruby -v").tap do |output|
-      assert_match "docker run --rm dhh/app:latest ruby -v", output
+      assert_match "docker run --rm --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env dhh/app:latest ruby -v", output
     end
   end
 
@@ -144,6 +170,25 @@ class CliAppTest < CliTestCase
     end
   end
 
+  test "exec interactive" do
+    SSHKit::Backend::Abstract.any_instance.expects(:exec)
+      .with("ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env dhh/app:latest ruby -v'")
+    run_command("exec", "-i", "ruby -v").tap do |output|
+      assert_match "Get most recent version available as an image...", output
+      assert_match "Launching interactive command with version latest via SSH from new container on 1.1.1.1...", output
+    end
+  end
+
+  test "exec interactive with reuse" do
+    SSHKit::Backend::Abstract.any_instance.expects(:exec)
+      .with("ssh -t root@1.1.1.1 -p 22 'docker exec -it app-web-999 ruby -v'")
+    run_command("exec", "-i", "--reuse", "ruby -v").tap do |output|
+      assert_match "Get current version of running container...", output
+      assert_match "Running docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done on 1.1.1.1", output
+      assert_match "Launching interactive command with version 999 via SSH from existing container on 1.1.1.1...", output
+    end
+  end
+
   test "containers" do
     run_command("containers").tap do |output|
       assert_match "docker container ls --all --filter label=service=app", output
@@ -165,7 +210,7 @@ class CliAppTest < CliTestCase
 
   test "logs with follow" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
+      .with("ssh -t root@1.1.1.1 -p 22 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
 
     assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
   end

test/cli/build_test.rb

@@ -57,6 +57,7 @@ class CliBuildTest < CliTestCase
     run_command("pull").tap do |output|
       assert_match /docker image rm --force dhh\/app:999/, output
       assert_match /docker pull dhh\/app:999/, output
+      assert_match "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:999 | grep -x app || (echo \"Image dhh/app:999 is missing the `service` label\" && exit 1)", output
     end
   end
 
@@ -66,6 +67,14 @@ class CliBuildTest < CliTestCase
     end
   end
 
+  test "create remote" do
+    run_command("create", fixture: :with_remote_builder).tap do |output|
+      assert_match "Running /usr/bin/env true on 1.1.1.5", output
+      assert_match "docker context create kamal-app-native-remote-amd64 --description 'kamal-app-native-remote amd64 native host' --docker 'host=ssh://app@1.1.1.5'", output
+      assert_match "docker buildx create --name kamal-app-native-remote kamal-app-native-remote-amd64 --platform linux/amd64", output
+    end
+  end
+
   test "create with error" do
     stub_setup
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
@@ -95,8 +104,8 @@ class CliBuildTest < CliTestCase
   end
 
   private
-    def run_command(*command)
+    def run_command(*command, fixture: :with_accessories)
-      stdouted { Kamal::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Build.start([*command, "-c", "test/fixtures/deploy_#{fixture}.yml"]) }
     end
 
     def stub_dependency_checks

test/cli/env_test.rb

@@ -9,25 +9,29 @@ class CliEnvTest < CliTestCase
       assert_match "Running /usr/bin/env mkdir -p .kamal/env/roles on 1.1.1.1", output
       assert_match "Running /usr/bin/env mkdir -p .kamal/env/traefik on 1.1.1.2", output
       assert_match "Running /usr/bin/env mkdir -p .kamal/env/accessories on 1.1.1.1", output
-      assert_match ".kamal/env/roles/app-web.env", output
+      assert_match ".kamal/env/roles/app-web-secret.env", output
-      assert_match ".kamal/env/roles/app-workers.env", output
+      assert_match ".kamal/env/roles/app-web-clear.env", output
-      assert_match ".kamal/env/traefik/traefik.env", output
+      assert_match ".kamal/env/roles/app-workers-secret.env", output
-      assert_match ".kamal/env/accessories/app-redis.env", output
+      assert_match ".kamal/env/roles/app-workers-clear.env", output
+      assert_match ".kamal/env/traefik/traefik-secret.env", output
+      assert_match ".kamal/env/traefik/traefik-clear.env", output
+      assert_match ".kamal/env/accessories/app-redis-secret.env", output
+      assert_match ".kamal/env/accessories/app-redis-clear.env", output
 
     end
   end
 
   test "delete" do
     run_command("delete").tap do |output|
-      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-web.env on 1.1.1.1", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-web*.env on 1.1.1.1", output
-      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-web.env on 1.1.1.2", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-web*.env on 1.1.1.2", output
-      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-workers.env on 1.1.1.3", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-workers*.env on 1.1.1.3", output
-      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-workers.env on 1.1.1.4", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-workers*.env on 1.1.1.4", output
-      assert_match "Running /usr/bin/env rm -f .kamal/env/traefik/traefik.env on 1.1.1.1", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/traefik/traefik*.env on 1.1.1.1", output
-      assert_match "Running /usr/bin/env rm -f .kamal/env/traefik/traefik.env on 1.1.1.2", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/traefik/traefik*.env on 1.1.1.2", output
-      assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-redis.env on 1.1.1.1", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-redis*.env on 1.1.1.1", output
-      assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-redis.env on 1.1.1.2", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-redis*.env on 1.1.1.2", output
-      assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-mysql.env on 1.1.1.3", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-mysql*.env on 1.1.1.3", output
     end
   end
 

test/cli/healthcheck_test.rb

@@ -5,13 +5,13 @@ class CliHealthcheckTest < CliTestCase
     # Prevent expected failures from outputting to terminal
     Thread.report_on_exception = false
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
     Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--env-file", ".kamal/env/roles/app-web.env", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
+      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--env-file", ".kamal/env/roles/app-web-secret.env", "--env-file", ".kamal/env/roles/app-web-clear.env", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false)
 
@@ -35,12 +35,12 @@ class CliHealthcheckTest < CliTestCase
     # Prevent expected failures from outputting to terminal
     Thread.report_on_exception = false
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--env-file", ".kamal/env/roles/app-web.env", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
+      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--env-file", ".kamal/env/roles/app-web-secret.env", "--env-file", ".kamal/env/roles/app-web-clear.env", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false)
 
@@ -65,8 +65,18 @@ class CliHealthcheckTest < CliTestCase
     assert_match "container not ready (unhealthy)", exception.message
   end
 
+  test "raises an exception if primary does not have traefik" do
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).never
+
+    exception = assert_raises do
+      run_command("perform", config_file: "test/fixtures/deploy_workers_only.yml")
+    end
+
+    assert_equal "The primary host is not configured to run Traefik", exception.message
+  end
+
   private
-    def run_command(*command)
+    def run_command(*command, config_file: "test/fixtures/deploy_with_accessories.yml")
-      stdouted { Kamal::Cli::Healthcheck.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Healthcheck.start([*command, "-c", config_file]) }
     end
 end

test/cli/main_test.rb

@@ -3,6 +3,7 @@ require_relative "cli_test_case"
 class CliMainTest < CliTestCase
   test "setup" do
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap")
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push")
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ])
     Kamal::Cli::Main.any_instance.expects(:deploy)
 
@@ -16,7 +17,7 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
@@ -43,7 +44,7 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
@@ -113,7 +114,7 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
@@ -122,9 +123,52 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "deploy without healthcheck if primary host doesn't have traefik" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_workers_only.yml", "version" => "999", "skip_hooks" => false }
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options).never
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
+
+    run_command("deploy", config_file: "deploy_workers_only")
+  end
+
   test "deploy with missing secrets" do
-    assert_raises(KeyError) do
+    invoke_options = { "config_file" => "test/fixtures/deploy_with_secrets.yml", "version" => "999", "skip_hooks" => false }
-      run_command("deploy", config_file: "deploy_with_secrets")
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
+
+    run_command("deploy", config_file: "deploy_with_secrets")
+  end
+
+  test "deploy with push_env" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_push_clear_env.yml", "version" => "999", "skip_hooks" => false }
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options.merge(env_type: "clear"))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
+
+    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+    hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "deploy" }
+
+    run_command("deploy", config_file: "deploy_push_clear_env").tap do |output|
+      assert_match /Pushing clear env files.../, output
     end
   end
 
@@ -133,7 +177,7 @@ class CliMainTest < CliTestCase
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
 
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
@@ -155,7 +199,7 @@ class CliMainTest < CliTestCase
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
 
     run_command("redeploy", "--skip_push").tap do |output|
@@ -164,6 +208,23 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "redeploy with push_env" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_push_clear_env.yml", "version" => "999", "skip_hooks" => false }
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options.merge(env_type: "clear"))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
+
+    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+    hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "redeploy" }
+
+    run_command("redeploy", config_file: "deploy_push_clear_env").tap do |output|
+      assert_match /Pushing clear env files.../, output
+    end
+  end
+
   test "rollback bad version" do
     Thread.report_on_exception = false
 
@@ -176,31 +237,8 @@ class CliMainTest < CliTestCase
   end
 
   test "rollback good version" do
-    Object.any_instance.stubs(:sleep)
+    stub_good_rollback
-    [ "web", "workers" ].each do |role|
-      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false)
-        .returns("").at_least_once
-      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet")
-        .returns("version-to-rollback\n").at_least_once
-      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-#{role}-}; done", raise_on_non_zero_exit: false)
-        .returns("version-to-rollback\n").at_least_once
-      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-        .returns("running").at_least_once # health check
-    end
 
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
-      .returns("corddirectory").at_least_once # health check
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-version-to-rollback$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-      .returns("unhealthy").at_least_once # health check
-
-    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     hook_variables = { version: 123, service_version: "app@123", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "rollback" }
 
     run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
@@ -215,7 +253,7 @@ class CliMainTest < CliTestCase
   test "rollback without old version" do
     Kamal::Cli::Main.any_instance.stubs(:container_available?).returns(true)
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep)
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep)
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
@@ -233,6 +271,16 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "rollback with push_env" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_push_clear_env.yml", "version" => "999", "skip_hooks" => false }
+
+    stub_good_rollback
+
+    run_command("rollback", "123", config_file: "deploy_push_clear_env").tap do |output|
+      assert_match /Pushing clear env files.../, output
+    end
+  end
+
   test "details" do
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:details")
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:details")
@@ -274,6 +322,16 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "config with primary web role override" do
+    run_command("config", config_file: "deploy_primary_web_role_override").tap do |output|
+      config = YAML.load(output)
+
+      assert_equal ["web_chicago", "web_tokyo"], config[:roles]
+      assert_equal ["1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4"], config[:hosts]
+      assert_equal "1.1.1.3", config[:primary_host]
+    end
+  end
+
   test "config with destination" do
     run_command("config", "-d", "world", config_file: "deploy_for_dest").tap do |output|
       config = YAML.load(output)
@@ -287,6 +345,19 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "config with aliases" do
+    run_command("config", config_file: "deploy_with_aliases").tap do |output|
+      config = YAML.load(output)
+
+      assert_equal ["web", "web_tokyo", "workers", "workers_tokyo"], config[:roles]
+      assert_equal ["1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4"], config[:hosts]
+      assert_equal "999", config[:version]
+      assert_equal "registry.digitalocean.com/dhh/app", config[:repository]
+      assert_equal "registry.digitalocean.com/dhh/app:999", config[:absolute_image]
+      assert_equal "app-999", config[:service_with_version]
+    end
+  end
+
   test "init" do
     Pathname.any_instance.expects(:exist?).returns(false).times(3)
     Pathname.any_instance.stubs(:mkpath)
@@ -345,6 +416,20 @@ class CliMainTest < CliTestCase
     run_command("envify")
   end
 
+  test "envify with blank line trimming" do
+    file = <<~EOF
+      HELLO=<%= 'world' %>
+      <% if true -%>
+      KEY=value
+      <% end  -%>
+    EOF
+
+    File.expects(:read).with(".env.erb").returns(file.strip)
+    File.expects(:write).with(".env", "HELLO=world\nKEY=value\n", perm: 0600)
+
+    run_command("envify")
+  end
+
   test "envify with destination" do
     File.expects(:read).with(".env.world.erb").returns("HELLO=<%= 'world' %>")
     File.expects(:write).with(".env.world", "HELLO=world", perm: 0600)
@@ -352,6 +437,14 @@ class CliMainTest < CliTestCase
     run_command("envify", "-d", "world", config_file: "deploy_for_dest")
   end
 
+  test "envify with skip_push" do
+    File.expects(:read).with(".env.erb").returns("HELLO=<%= 'world' %>")
+    File.expects(:write).with(".env", "HELLO=world", perm: 0600)
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push").never
+    run_command("envify", "--skip-push")
+  end
+
   test "remove with confirmation" do
     run_command("remove", "-y", config_file: "deploy_with_accessories").tap do |output|
       assert_match /docker container stop traefik/, output
@@ -385,4 +478,32 @@ class CliMainTest < CliTestCase
     def run_command(*command, config_file: "deploy_simple")
       stdouted { Kamal::Cli::Main.start([*command, "-c", "test/fixtures/#{config_file}.yml"]) }
     end
+
+    def stub_good_rollback
+      Object.any_instance.stubs(:sleep)
+      [ "web", "workers" ].each do |role|
+        SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+          .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false)
+          .returns("").at_least_once
+        SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+          .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet")
+          .returns("version-to-rollback\n").at_least_once
+        SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+          .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-#{role}-}; done", raise_on_non_zero_exit: false)
+          .returns("version-to-rollback\n").at_least_once
+        SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+          .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+          .returns("running").at_least_once # health check
+      end
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+        .returns("corddirectory").at_least_once # health check
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-version-to-rollback$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+        .returns("unhealthy").at_least_once # health check
+
+      Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+    end
 end

test/cli/prune_test.rb

@@ -10,7 +10,7 @@ class CliPruneTest < CliTestCase
 
   test "images" do
     run_command("images").tap do |output|
-      assert_match "docker image prune --force --filter label=service=app --filter dangling=true on 1.1.1.", output
+      assert_match "docker image prune --force --filter label=service=app on 1.1.1.", output
       assert_match "docker image ls --filter label=service=app --format '{{.ID}} {{.Repository}}:{{.Tag}}' | grep -v -w \"$(docker container ls -a --format '{{.Image}}\\|' --filter label=service=app | tr -d '\\n')dhh/app:latest\\|dhh/app:<none>\" | while read image tag; do docker rmi $tag; done on 1.1.1.", output
     end
   end
@@ -18,6 +18,16 @@ class CliPruneTest < CliTestCase
   test "containers" do
     run_command("containers").tap do |output|
       assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output
+      assert_match /docker container prune --force --filter label=service=healthcheck-app on 1.1.1.\d/, output
+     end
+
+    run_command("containers", "--retain", "10").tap do |output|
+      assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +11 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output
+      assert_match /docker container prune --force --filter label=service=healthcheck-app on 1.1.1.\d/, output
+    end
+
+    assert_raises(RuntimeError, "retain must be at least 1") do
+      run_command("containers", "--retain", "0")
     end
   end
 

test/cli/server_test.rb

@@ -10,7 +10,7 @@ class CliServerTest < CliTestCase
 
   test "bootstrap install as non-root user" do
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ]', raise_on_non_zero_exit: false).returns(false).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', raise_on_non_zero_exit: false).returns(false).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
 
     assert_raise RuntimeError, "Docker is not installed on 1.1.1.1, 1.1.1.3, 1.1.1.4, 1.1.1.2 and can't be automatically installed without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/" do
@@ -20,7 +20,7 @@ class CliServerTest < CliTestCase
 
   test "bootstrap install as root user" do
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ]', raise_on_non_zero_exit: false).returns(true).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', raise_on_non_zero_exit: false).returns(true).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:curl, "-fsSL", "https://get.docker.com", "|", :sh).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
 

test/cli/traefik_test.rb

@@ -4,7 +4,7 @@ class CliTraefikTest < CliTestCase
   test "boot" do
     run_command("boot").tap do |output|
       assert_match "docker login", output
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
+      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
     end
   end
 
@@ -14,7 +14,7 @@ class CliTraefikTest < CliTestCase
     run_command("reboot").tap do |output|
       assert_match "docker container stop traefik", output
       assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik", output
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
+      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
     end
   end
 
@@ -64,7 +64,7 @@ class CliTraefikTest < CliTestCase
 
   test "logs with follow" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 'docker logs traefik --timestamps --tail 10 --follow 2>&1'")
+      .with("ssh -t root@1.1.1.1 -p 22 'docker logs traefik --timestamps --tail 10 --follow 2>&1'")
 
     assert_match "docker logs traefik --timestamps --tail 10 --follow", run_command("logs", "--follow")
   end

test/commander_test.rb

@@ -14,6 +14,20 @@ class CommanderTest < ActiveSupport::TestCase
 
     @kamal.specific_hosts = [ "1.1.1.1", "1.1.1.2" ]
     assert_equal [ "1.1.1.1", "1.1.1.2" ], @kamal.hosts
+
+    @kamal.specific_hosts = [ "1.1.1.1*" ]
+    assert_equal [ "1.1.1.1" ], @kamal.hosts
+
+    @kamal.specific_hosts = [ "1.1.1.*", "*.1.2.*" ]
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @kamal.hosts
+
+    @kamal.specific_hosts = [ "*" ]
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @kamal.hosts
+
+    exception = assert_raises(ArgumentError) do
+      @kamal.specific_hosts = [ "*miss" ]
+    end
+    assert_match /hosts match for \*miss/, exception.message
   end
 
   test "filtering hosts by filtering roles" do
@@ -21,6 +35,11 @@ class CommanderTest < ActiveSupport::TestCase
 
     @kamal.specific_roles = [ "web" ]
     assert_equal [ "1.1.1.1", "1.1.1.2" ], @kamal.hosts
+
+    exception = assert_raises(ArgumentError) do
+      @kamal.specific_roles = [ "*miss" ]
+    end
+    assert_match /roles match for \*miss/, exception.message
   end
 
   test "filtering roles" do
@@ -28,6 +47,20 @@ class CommanderTest < ActiveSupport::TestCase
 
     @kamal.specific_roles = [ "workers" ]
     assert_equal [ "workers" ], @kamal.roles.map(&:name)
+
+    @kamal.specific_roles = [ "w*" ]
+    assert_equal [ "web", "workers" ], @kamal.roles.map(&:name)
+
+    @kamal.specific_roles = [ "we*", "*orkers" ]
+    assert_equal [ "web", "workers" ], @kamal.roles.map(&:name)
+
+    @kamal.specific_roles = [ "*" ]
+    assert_equal [ "web", "workers" ], @kamal.roles.map(&:name)
+
+    exception = assert_raises(ArgumentError) do
+      @kamal.specific_roles = [ "*miss" ]
+    end
+    assert_match /roles match for \*miss/, exception.message
   end
 
   test "filtering roles by filtering hosts" do
@@ -49,6 +82,12 @@ class CommanderTest < ActiveSupport::TestCase
     assert_equal "1.1.1.3", @kamal.primary_host
   end
 
+  test "primary_role" do
+    assert_equal "web", @kamal.primary_role
+    @kamal.specific_roles = "workers"
+    assert_equal "workers", @kamal.primary_role
+  end
+
   test "roles_on" do
     assert_equal [ "web" ], @kamal.roles_on("1.1.1.1")
     assert_equal [ "workers" ], @kamal.roles_on("1.1.1.3")
@@ -70,6 +109,15 @@ class CommanderTest < ActiveSupport::TestCase
     assert_equal({ in: :groups, limit: 1, wait: 2 }, @kamal.boot_strategy)
   end
 
+  test "try to match the primary role from a list of specific roles" do
+    configure_with(:deploy_primary_web_role_override)
+
+    @kamal.specific_roles = [ "web_*" ]
+    assert_equal [ "web_chicago", "web_tokyo" ], @kamal.roles.map(&:name)
+    assert_equal "web_tokyo", @kamal.primary_role
+    assert_equal "1.1.1.3", @kamal.primary_host
+  end
+
   private
     def configure_with(variant)
       @kamal = Kamal::Commander.new.tap do |kamal|

test/commands/accessory_test.rb

@@ -34,6 +34,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
           ]
         },
         "busybox" => {
+          "service" => "custom-busybox",
           "image" => "busybox:latest",
           "host" => "1.1.1.7"
         }
@@ -49,15 +50,15 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --label service=\"app-mysql\" private.registry/mysql:8.0",
+      "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql-secret.env --env-file .kamal/env/accessories/app-mysql-clear.env --label service=\"app-mysql\" private.registry/mysql:8.0",
       new_command(:mysql).run.join(" ")
 
     assert_equal \
-      "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest",
+      "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis-secret.env --env-file .kamal/env/accessories/app-redis-clear.env --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest",
       new_command(:redis).run.join(" ")
 
     assert_equal \
-      "docker run --name app-busybox --detach --restart unless-stopped --log-opt max-size=\"10m\" --env-file .kamal/env/accessories/app-busybox.env --label service=\"app-busybox\" busybox:latest",
+      "docker run --name custom-busybox --detach --restart unless-stopped --log-opt max-size=\"10m\" --env-file .kamal/env/accessories/custom-busybox-secret.env --env-file .kamal/env/accessories/custom-busybox-clear.env --label service=\"custom-busybox\" busybox:latest",
       new_command(:busybox).run.join(" ")
   end
 
@@ -65,7 +66,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --name app-busybox --detach --restart unless-stopped --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --env-file .kamal/env/accessories/app-busybox.env --label service=\"app-busybox\" busybox:latest",
+      "docker run --name custom-busybox --detach --restart unless-stopped --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --env-file .kamal/env/accessories/custom-busybox-secret.env --env-file .kamal/env/accessories/custom-busybox-clear.env --label service=\"custom-busybox\" busybox:latest",
       new_command(:busybox).run.join(" ")
   end
 
@@ -90,7 +91,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "execute in new container" do
     assert_equal \
-      "docker run --rm --env-file .kamal/env/accessories/app-mysql.env private.registry/mysql:8.0 mysql -u root",
+      "docker run --rm --env-file .kamal/env/accessories/app-mysql-secret.env --env-file .kamal/env/accessories/app-mysql-clear.env private.registry/mysql:8.0 mysql -u root",
       new_command(:mysql).execute_in_new_container("mysql", "-u", "root").join(" ")
   end
 
@@ -102,7 +103,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "execute in new container over ssh" do
     new_command(:mysql).stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do
-      assert_match %r|docker run -it --rm --env-file .kamal/env/accessories/app-mysql.env private.registry/mysql:8.0 mysql -u root|,
+      assert_match %r|docker run -it --rm --env-file .kamal/env/accessories/app-mysql-secret.env --env-file .kamal/env/accessories/app-mysql-clear.env private.registry/mysql:8.0 mysql -u root|,
         new_command(:mysql).execute_in_new_container_over_ssh("mysql", "-u", "root")
     end
   end
@@ -128,7 +129,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "follow logs" do
     assert_equal \
-      "ssh -t root@1.1.1.5 'docker logs app-mysql --timestamps --tail 10 --follow 2>&1'",
+      "ssh -t root@1.1.1.5 -p 22 'docker logs app-mysql --timestamps --tail 10 --follow 2>&1'",
       new_command(:mysql).follow_logs
   end
 
@@ -148,8 +149,8 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
     assert_equal "mkdir -p .kamal/env/accessories", new_command(:mysql).make_env_directory.join(" ")
   end
 
-  test "remove_env_file" do
+  test "remove_env_files" do
-    assert_equal "rm -f .kamal/env/accessories/app-mysql.env", new_command(:mysql).remove_env_file.join(" ")
+    assert_equal "rm -f .kamal/env/accessories/app-mysql*.env", new_command(:mysql).remove_env_files.join(" ")
   end
 
   private

test/commands/app_test.rb

@@ -14,13 +14,13 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with hostname" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run(hostname: "myhost").join(" ")
   end
 
@@ -28,7 +28,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:volumes] = ["/local/path:/container/path" ]
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -36,7 +36,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "path" => "/healthz" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -44,7 +44,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "cmd" => "/bin/up" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -52,14 +52,14 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with custom options" do
     @config[:servers] = { "web" => [ "1.1.1.1" ], "jobs" => { "hosts" => [ "1.1.1.2" ], "cmd" => "bin/jobs", "options" => { "mount" => "somewhere", "cap-add" => true } } }
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" --env-file .kamal/env/roles/app-jobs.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
+      "docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-jobs-secret.env --env-file .kamal/env/roles/app-jobs-clear.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
       new_command(role: "jobs").run.join(" ")
   end
 
@@ -67,7 +67,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -156,14 +156,14 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "execute in new container" do
     assert_equal \
-      "docker run --rm --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails db:setup",
+      "docker run --rm --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env dhh/app:999 bin/rails db:setup",
       new_command.execute_in_new_container("bin/rails", "db:setup").join(" ")
   end
 
   test "execute in new container with custom options" do
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } }
     assert_equal \
-      "docker run --rm --env-file .kamal/env/roles/app-web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails db:setup",
+      "docker run --rm --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails db:setup",
       new_command.execute_in_new_container("bin/rails", "db:setup").join(" ")
   end
 
@@ -174,13 +174,13 @@ class CommandsAppTest < ActiveSupport::TestCase
   end
 
   test "execute in new container over ssh" do
-    assert_match %r|docker run -it --rm --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails c|,
+    assert_match %r|docker run -it --rm --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env dhh/app:999 bin/rails c|,
       new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1")
   end
 
   test "execute in new container with custom options over ssh" do
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } }
-    assert_match %r|docker run -it --rm --env-file .kamal/env/roles/app-web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails c|,
+    assert_match %r|docker run -it --rm --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails c|,
       new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1")
   end
 
@@ -190,32 +190,37 @@ class CommandsAppTest < ActiveSupport::TestCase
   end
 
   test "run over ssh" do
-    assert_equal "ssh -t root@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with custom user" do
     @config[:ssh] = { "user" => "app" }
-    assert_equal "ssh -t app@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -t app@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+  end
+
+  test "run over ssh with custom port" do
+    @config[:ssh] = { "port" => "2222" }
+    assert_equal "ssh -t root@1.1.1.1 -p 2222 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with proxy" do
     @config[:ssh] = { "proxy" => "2.2.2.2" }
-    assert_equal "ssh -J root@2.2.2.2 -t root@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -J root@2.2.2.2 -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with proxy user" do
     @config[:ssh] = { "proxy" => "app@2.2.2.2" }
-    assert_equal "ssh -J app@2.2.2.2 -t root@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -J app@2.2.2.2 -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with custom user with proxy" do
     @config[:ssh] = { "user" => "app", "proxy" => "2.2.2.2" }
-    assert_equal "ssh -J root@2.2.2.2 -t app@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -J root@2.2.2.2 -t app@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with proxy_command" do
     @config[:ssh] = { "proxy_command" => "ssh -W %h:%p user@proxy-server" }
-    assert_equal "ssh -o ProxyCommand='ssh -W %h:%p user@proxy-server' -t root@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -o ProxyCommand='ssh -W %h:%p user@proxy-server' -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "current_running_container_id" do
@@ -317,10 +322,10 @@ class CommandsAppTest < ActiveSupport::TestCase
       new_command.remove_images.join(" ")
   end
 
-  test "tag_current_as_latest" do
+  test "tag_current_image_as_latest" do
     assert_equal \
       "docker tag dhh/app:999 dhh/app:latest",
-      new_command.tag_current_as_latest.join(" ")
+      new_command.tag_current_image_as_latest.join(" ")
   end
 
   test "make_env_directory" do
@@ -328,11 +333,11 @@ class CommandsAppTest < ActiveSupport::TestCase
   end
 
   test "remove_env_file" do
-    assert_equal "rm -f .kamal/env/roles/app-web.env", new_command.remove_env_file.join(" ")
+    assert_equal "rm -f .kamal/env/roles/app-web*.env", new_command.remove_env_files.join(" ")
   end
 
   test "cord" do
-    assert_equal "docker inspect -f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}' app-web-123 | awk '$2 == \"/tmp/kamal-cord\" {print $1}'", new_command.cord(version: 123).join(" ")
+    assert_equal "docker inspect -f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}' app-web-123 | awk '$2 == \"/tmp/kamal-cord\" {print $1}'", new_command.cord(version: 123).join(" ")
   end
 
   test "tie cord" do
@@ -345,8 +350,39 @@ class CommandsAppTest < ActiveSupport::TestCase
     assert_equal "rm -r corddir", new_command.cut_cord("corddir").join(" ")
   end
 
+  test "extract assets" do
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/extracted/app-web-999", "&&",
+      :docker, :stop, "-t 1", "app-web-assets", "2> /dev/null", "|| true", "&&",
+      :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "dhh/app:latest", "sleep 1000000", "&&",
+      :docker, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/assets/extracted/app-web-999", "&&",
+      :docker, :stop, "-t 1", "app-web-assets"
+    ], new_command(asset_path: "/public/assets").extract_assets
+  end
+
+  test "sync asset volumes" do
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-999", ".kamal/assets/volumes/app-web-999"
+    ], new_command(asset_path: "/public/assets").sync_asset_volumes
+
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-999", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-999", ".kamal/assets/volumes/app-web-998", "|| true", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-998", ".kamal/assets/volumes/app-web-999", "|| true",
+    ], new_command(asset_path: "/public/assets").sync_asset_volumes(old_version: 998)
+  end
+
+  test "clean up assets" do
+    assert_equal [
+      :find, ".kamal/assets/extracted", "-maxdepth 1", "-name", "'app-web-*'", "!", "-name", "app-web-999", "-exec rm -rf \"{}\" +", ";",
+      :find, ".kamal/assets/volumes", "-maxdepth 1", "-name", "'app-web-*'", "!", "-name", "app-web-999", "-exec rm -rf \"{}\" +"
+    ], new_command(asset_path: "/public/assets").clean_up_assets
+  end
+
   private
-    def new_command(role: "web")
+    def new_command(role: "web", **additional_config)
-      Kamal::Commands::App.new(Kamal::Configuration.new(@config, destination: @destination, version: "999"), role: role)
+      Kamal::Commands::App.new(Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999"), role: role)
     end
 end

test/commands/builder_test.rb

@@ -37,6 +37,14 @@ class CommandsBuilderTest < ActiveSupport::TestCase
       builder.push.join(" ")
   end
 
+  test "target multiarch local when arch is set" do
+    builder = new_builder_command(builder: { "local" => { "arch" => "amd64" } })
+    assert_equal "multiarch", builder.name
+    assert_equal \
+      "docker buildx build --push --platform linux/amd64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .",
+      builder.push.join(" ")
+  end
+
   test "target native remote when only remote is set" do
     builder = new_builder_command(builder: { "remote" => { "arch" => "amd64" }, "cache" => { "type" => "gha" } })
     assert_equal "native/remote", builder.name
@@ -103,6 +111,18 @@ class CommandsBuilderTest < ActiveSupport::TestCase
       builder.push.join(" ")
   end
 
+  test "build with ssh agent socket" do
+    builder = new_builder_command(builder: { "ssh" => 'default=$SSH_AUTH_SOCK' })
+
+    assert_equal \
+      "-t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --ssh default=$SSH_AUTH_SOCK",
+      builder.target.build_options.join(" ")
+  end
+
+  test "validate image" do
+    assert_equal "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:123 | grep -x app || (echo \"Image dhh/app:123 is missing the `service` label\" && exit 1)", new_builder_command.validate_image.join(" ")
+  end
+
   private
     def new_builder_command(additional_config = {})
       Kamal::Commands::Builder.new(Kamal::Configuration.new(@config.merge(additional_config), version: "123"))

test/commands/docker_test.rb

@@ -21,6 +21,6 @@ class CommandsDockerTest < ActiveSupport::TestCase
   end
 
   test "superuser?" do
-    assert_equal '[ "${EUID:-$(id -u)}" -eq 0 ]', @docker.superuser?.join(" ")
+    assert_equal '[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', @docker.superuser?.join(" ")
   end
 end

test/commands/healthcheck_test.rb

@@ -10,7 +10,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
       new_command.run.join(" ")
   end
 
@@ -18,7 +18,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "port" => 3001 }
 
     assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3001 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3001/up || exit 1\" --health-interval \"1s\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 3999:3001 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env --health-cmd \"curl -f http://localhost:3001/up || exit 1\" --health-interval \"1s\" dhh/app:123",
       new_command.run.join(" ")
   end
 
@@ -26,7 +26,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
     @destination = "staging"
 
     assert_equal \
-      "docker run --detach --name healthcheck-app-staging-123 --publish 3999:3000 --label service=healthcheck-app-staging -e KAMAL_CONTAINER_NAME=\"healthcheck-app-staging\" --env-file .kamal/env/roles/app-web-staging.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-staging-123 --publish 3999:3000 --label service=healthcheck-app-staging -e KAMAL_CONTAINER_NAME=\"healthcheck-app-staging\" --env-file .kamal/env/roles/app-web-staging-secret.env --env-file .kamal/env/roles/app-web-staging-clear.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
       new_command.run.join(" ")
   end
 
@@ -34,7 +34,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "cmd" => "/bin/up" }
 
     assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/up\" --health-interval \"1s\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env --health-cmd \"/bin/up\" --health-interval \"1s\" dhh/app:123",
       new_command.run.join(" ")
   end
 
@@ -42,7 +42,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere" } } }
     @config[:healthcheck] = { "exposed_port" => 4999 }
     assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 4999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 4999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web-secret.env --env-file .kamal/env/roles/app-web-clear.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
       new_command.run.join(" ")
   end
 
@@ -92,6 +92,13 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
       new_command.logs.join(" ")
   end
 
+  test "logs with custom lines number" do
+    @config[:healthcheck] = { "log_lines" => 150 }
+    assert_equal \
+      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker logs --tail 150 2>&1",
+      new_command.logs.join(" ")
+  end
+
   test "logs with destination" do
     @destination = "staging"
 

test/commands/prune_test.rb

@@ -10,7 +10,7 @@ class CommandsPruneTest < ActiveSupport::TestCase
 
   test "dangling images" do
     assert_equal \
-      "docker image prune --force --filter label=service=app --filter dangling=true",
+      "docker image prune --force --filter label=service=app",
       new_command.dangling_images.join(" ")
   end
 
@@ -20,10 +20,20 @@ class CommandsPruneTest < ActiveSupport::TestCase
       new_command.tagged_images.join(" ")
   end
 
-  test "containers" do
+  test "app containers" do
     assert_equal \
       "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done",
-      new_command.containers.join(" ")
+      new_command.app_containers(retain: 5).join(" ")
+
+    assert_equal \
+      "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +4 | while read container_id; do docker rm $container_id; done",
+      new_command.app_containers(retain: 3).join(" ")
+  end
+
+  test "healthcheck containers" do
+    assert_equal \
+      "docker container prune --force --filter label=service=healthcheck-app",
+      new_command.healthcheck_containers.join(" ")
   end
 
   private

test/commands/traefik_test.rb

@@ -18,72 +18,72 @@ class CommandsTraefikTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["host_port"] = "8080"
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["publish"] = false
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with ports configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["options"] = {"publish" => %w[9000:9000 9001:9001]}
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with volumes configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json] }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with several options configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json], "publish" => %w[8080:8080], "memory" => "512m"}
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with labels configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["labels"] = { "traefik.http.routers.dashboard.service" => "api@internal", "traefik.http.routers.dashboard.middlewares" => "auth" }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.dashboard.service=\"api@internal\" --label traefik.http.routers.dashboard.middlewares=\"auth\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --label traefik.http.routers.dashboard.service=\"api@internal\" --label traefik.http.routers.dashboard.middlewares=\"auth\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with env configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -91,7 +91,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
     @config.delete(:traefik)
 
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"",
       new_command.run.join(" ")
   end
 
@@ -99,7 +99,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -107,7 +107,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
     @config[:traefik]["args"]["log.level"] = "ERROR"
 
     assert_equal \
-    "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"ERROR\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+    "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik-secret.env --env-file .kamal/env/traefik/traefik-clear.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"ERROR\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -167,32 +167,36 @@ class CommandsTraefikTest < ActiveSupport::TestCase
 
   test "traefik follow logs" do
     assert_equal \
-      "ssh -t root@1.1.1.1 'docker logs traefik --timestamps --tail 10 --follow 2>&1'",
+      "ssh -t root@1.1.1.1 -p 22 'docker logs traefik --timestamps --tail 10 --follow 2>&1'",
       new_command.follow_logs(host: @config[:servers].first)
   end
 
   test "traefik follow logs with grep hello!" do
     assert_equal \
-      "ssh -t root@1.1.1.1 'docker logs traefik --timestamps --tail 10 --follow 2>&1 | grep \"hello!\"'",
+      "ssh -t root@1.1.1.1 -p 22 'docker logs traefik --timestamps --tail 10 --follow 2>&1 | grep \"hello!\"'",
       new_command.follow_logs(host: @config[:servers].first, grep: 'hello!')
   end
 
   test "env_file" do
     @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
 
-    assert_equal "EXAMPLE_API_KEY=456\n", new_command.env_file
+    assert_equal "EXAMPLE_API_KEY=456\n", new_command.env_file.secret
   end
 
-  test "host_env_file_path" do
+  test "host_secret_env_file_path" do
-    assert_equal ".kamal/env/traefik/traefik.env", new_command.host_env_file_path
+    assert_equal ".kamal/env/traefik/traefik-secret.env", new_command.host_secret_env_file_path
+  end
+
+  test "host_clear_env_file_path" do
+    assert_equal ".kamal/env/traefik/traefik-clear.env", new_command.host_clear_env_file_path
   end
 
   test "make_env_directory" do
     assert_equal "mkdir -p .kamal/env/traefik", new_command.make_env_directory.join(" ")
   end
 
-  test "remove_env_file" do
+  test "remove_env_files" do
-    assert_equal "rm -f .kamal/env/traefik/traefik.env", new_command.remove_env_file.join(" ")
+    assert_equal "rm -f .kamal/env/traefik/traefik*.env", new_command.remove_env_files.join(" ")
   end
 
   private

test/configuration/accessory_test.rb

@@ -49,6 +49,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
           }
         },
         "monitoring" => {
+          "service" => "custom-monitoring",
           "image" => "monitoring:latest",
           "roles" => [ "web" ],
           "port" => "4321:4321",
@@ -72,6 +73,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
   test "service name" do
     assert_equal "app-mysql", @config.accessory(:mysql).service_name
     assert_equal "app-redis", @config.accessory(:redis).service_name
+    assert_equal "custom-monitoring", @config.accessory(:monitoring).service_name
   end
 
   test "port" do
@@ -111,19 +113,27 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
   end
 
   test "env args" do
-    assert_equal ["--env-file", ".kamal/env/accessories/app-mysql.env"], @config.accessory(:mysql).env_args
+    assert_equal \
-    assert_equal ["--env-file", ".kamal/env/accessories/app-redis.env"], @config.accessory(:redis).env_args
+      ["--env-file", ".kamal/env/accessories/app-mysql-secret.env", "--env-file", ".kamal/env/accessories/app-mysql-clear.env"],
+      @config.accessory(:mysql).env_args
+    assert_equal \
+      ["--env-file", ".kamal/env/accessories/app-redis-secret.env", "--env-file", ".kamal/env/accessories/app-redis-clear.env"],
+      @config.accessory(:redis).env_args
   end
 
   test "env file with secret" do
     ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
 
-    expected = <<~ENV
+    expected_secret = <<~ENV
       MYSQL_ROOT_PASSWORD=secret123
+    ENV
+
+    expected_clear = <<~ENV
       MYSQL_ROOT_HOST=%
     ENV
 
-    assert_equal expected, @config.accessory(:mysql).env_file
+    assert_equal expected_secret, @config.accessory(:mysql).env_file.secret
+    assert_equal expected_clear, @config.accessory(:mysql).env_file.clear
   ensure
     ENV["MYSQL_ROOT_PASSWORD"] = nil
   end
@@ -132,8 +142,12 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
     assert_equal ".kamal/env/accessories", @config.accessory(:mysql).host_env_directory
   end
 
-  test "host_env_file_path" do
+  test "host_secret_env_file_path" do
-    assert_equal ".kamal/env/accessories/app-mysql.env", @config.accessory(:mysql).host_env_file_path
+    assert_equal ".kamal/env/accessories/app-mysql-secret.env", @config.accessory(:mysql).host_secret_env_file_path
+  end
+
+  test "host_clear_env_file_path" do
+    assert_equal ".kamal/env/accessories/app-mysql-clear.env", @config.accessory(:mysql).host_clear_env_file_path
   end
 
   test "volume args" do
@@ -149,10 +163,16 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
     assert_match "%", @config.accessory(:mysql).files.keys[2].read
   end
 
-  test "directories" do
+  test "directory with a relative path" do
+    @deploy[:accessories]["mysql"]["directories"] = [ "data:/var/lib/mysql" ]
     assert_equal({"$PWD/app-mysql/data"=>"/var/lib/mysql"}, @config.accessory(:mysql).directories)
   end
 
+  test "directory with an absolute path" do
+    @deploy[:accessories]["mysql"]["directories"] = [ "/var/data/mysql:/var/lib/mysql" ]
+    assert_equal({"/var/data/mysql"=>"/var/lib/mysql"}, @config.accessory(:mysql).directories)
+  end
+
   test "options" do
     assert_equal ["--cpus", "\"4\"", "--memory", "\"2GB\""], @config.accessory(:redis).option_args
   end

test/configuration/builder_test.rb

@@ -148,4 +148,14 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase
 
     assert_equal "..", @config_with_builder_option.builder.context
   end
+
+  test "ssh" do
+    assert_nil @config.builder.ssh
+  end
+
+  test "setting ssh params" do
+    @deploy_with_builder_option[:builder] = { "ssh" => 'default=$SSH_AUTH_SOCK' }
+
+    assert_equal 'default=$SSH_AUTH_SOCK', @config_with_builder_option.builder.ssh
+  end
 end

test/configuration/role_test.rb

@@ -42,7 +42,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
   end
 
   test "special label args for web" do
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.routers.app-web.priority=\"2\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args
   end
 
   test "custom labels" do
@@ -66,7 +66,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       c[:servers]["beta"] = { "traefik" => "true", "hosts" => [ "1.1.1.5" ] }
     })
 
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "traefik.http.services.app-beta.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "traefik.http.services.app-beta.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.routers.app-beta.priority=\"2\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args
   end
 
   test "env overwritten by role" do
@@ -77,11 +77,23 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected_env, @config_with_roles.role(:workers).env_file
+    assert_equal "\n", @config_with_roles.role(:workers).env_file.secret
+    assert_equal expected_env, @config_with_roles.role(:workers).env_file.clear
+  end
+
+  test "container name" do
+    ENV["VERSION"] = "12345"
+
+    assert_equal "app-workers-12345", @config_with_roles.role(:workers).container_name
+    assert_equal "app-web-12345", @config_with_roles.role(:web).container_name
+  ensure
+    ENV.delete("VERSION")
   end
 
   test "env args" do
-    assert_equal ["--env-file", ".kamal/env/roles/app-workers.env"], @config_with_roles.role(:workers).env_args
+    assert_equal \
+      ["--env-file", ".kamal/env/roles/app-workers-secret.env", "--env-file", ".kamal/env/roles/app-workers-clear.env"],
+      @config_with_roles.role(:workers).env_args
   end
 
   test "env secret overwritten by role" do
@@ -107,14 +119,18 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     ENV["REDIS_PASSWORD"] = "secret456"
     ENV["DB_PASSWORD"] = "secret&\"123"
 
-    expected = <<~ENV
+    expected_secret = <<~ENV
       REDIS_PASSWORD=secret456
       DB_PASSWORD=secret&\"123
+    ENV
+
+    expected_clear = <<~ENV
       REDIS_URL=redis://a/b
       WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected, @config_with_roles.role(:workers).env_file
+    assert_equal expected_secret, @config_with_roles.role(:workers).env_file.secret
+    assert_equal expected_clear, @config_with_roles.role(:workers).env_file.clear
   ensure
     ENV["REDIS_PASSWORD"] = nil
     ENV["DB_PASSWORD"] = nil
@@ -133,13 +149,17 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
 
     ENV["DB_PASSWORD"] = "secret123"
 
-    expected = <<~ENV
+    expected_secret = <<~ENV
       DB_PASSWORD=secret123
+    ENV
+
+    expected_clear = <<~ENV
       REDIS_URL=redis://a/b
       WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected, @config_with_roles.role(:workers).env_file
+    assert_equal expected_secret, @config_with_roles.role(:workers).env_file.secret
+    assert_equal expected_clear, @config_with_roles.role(:workers).env_file.clear
   ensure
     ENV["DB_PASSWORD"] = nil
   end
@@ -156,13 +176,49 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
 
     ENV["REDIS_PASSWORD"] = "secret456"
 
-    expected = <<~ENV
+    expected_secret = <<~ENV
       REDIS_PASSWORD=secret456
+    ENV
+
+    expected_clear = <<~ENV
       REDIS_URL=redis://a/b
       WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected, @config_with_roles.role(:workers).env_file
+    assert_equal expected_secret, @config_with_roles.role(:workers).env_file.secret
+    assert_equal expected_clear, @config_with_roles.role(:workers).env_file.clear
+  ensure
+    ENV["REDIS_PASSWORD"] = nil
+  end
+
+  test "env overwritten by role with secrets" do
+    @deploy_with_roles[:env] = {
+      "clear" => {
+        "REDIS_URL" => "redis://a/b"
+      },
+      "secret" => [
+        "REDIS_PASSWORD"
+      ]
+    }
+
+    @deploy_with_roles[:servers]["workers"]["env"] = {
+      "clear" => {
+        "REDIS_URL" => "redis://c/d",
+      },
+    }
+
+    ENV["REDIS_PASSWORD"] = "secret456"
+
+    expected_secret = <<~ENV
+      REDIS_PASSWORD=secret456
+    ENV
+
+    expected_clear = <<~ENV
+      REDIS_URL=redis://c/d
+    ENV
+
+    assert_equal expected_secret, @config_with_roles.role(:workers).env_file.secret
+    assert_equal expected_clear, @config_with_roles.role(:workers).env_file.clear
   ensure
     ENV["REDIS_PASSWORD"] = nil
   end
@@ -171,8 +227,12 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     assert_equal ".kamal/env/roles", @config_with_roles.role(:workers).host_env_directory
   end
 
-  test "host_env_file_path" do
+  test "host_secret_env_file_path" do
-    assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).host_env_file_path
+    assert_equal ".kamal/env/roles/app-workers-secret.env", @config_with_roles.role(:workers).host_secret_env_file_path
+  end
+
+  test "host_clear_env_file_path" do
+    assert_equal ".kamal/env/roles/app-workers-clear.env", @config_with_roles.role(:workers).host_clear_env_file_path
   end
 
   test "uses cord" do
@@ -194,4 +254,53 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
   test "cord container file" do
     assert_equal "/tmp/kamal-cord/cord", @config_with_roles.role(:web).cord_container_file
   end
+
+  test "asset path and volume args" do
+    ENV["VERSION"] = "12345"
+    assert_nil @config_with_roles.role(:web).asset_volume_args
+    assert_nil @config_with_roles.role(:workers).asset_volume_args
+    assert_nil @config_with_roles.role(:web).asset_path
+    assert_nil @config_with_roles.role(:workers).asset_path
+    assert !@config_with_roles.role(:web).assets?
+    assert !@config_with_roles.role(:workers).assets?
+
+    config_with_assets = Kamal::Configuration.new(@deploy_with_roles.dup.tap { |c|
+      c[:asset_path] = "foo"
+    })
+    assert_equal "foo", config_with_assets.role(:web).asset_path
+    assert_equal "foo", config_with_assets.role(:workers).asset_path
+    assert_equal ["--volume", "$(pwd)/.kamal/assets/volumes/app-web-12345:foo"], config_with_assets.role(:web).asset_volume_args
+    assert_nil config_with_assets.role(:workers).asset_volume_args
+    assert config_with_assets.role(:web).assets?
+    assert !config_with_assets.role(:workers).assets?
+
+    config_with_assets = Kamal::Configuration.new(@deploy_with_roles.dup.tap { |c|
+      c[:servers]["web"] = { "hosts" => [ "1.1.1.1", "1.1.1.2" ], "asset_path" => "bar" }
+    })
+    assert_equal "bar", config_with_assets.role(:web).asset_path
+    assert_nil config_with_assets.role(:workers).asset_path
+    assert_equal ["--volume", "$(pwd)/.kamal/assets/volumes/app-web-12345:bar"], config_with_assets.role(:web).asset_volume_args
+    assert_nil config_with_assets.role(:workers).asset_volume_args
+    assert config_with_assets.role(:web).assets?
+    assert !config_with_assets.role(:workers).assets?
+
+  ensure
+    ENV.delete("VERSION")
+  end
+
+  test "asset extracted path" do
+    ENV["VERSION"] = "12345"
+    assert_equal ".kamal/assets/extracted/app-web-12345", @config_with_roles.role(:web).asset_extracted_path
+    assert_equal ".kamal/assets/extracted/app-workers-12345", @config_with_roles.role(:workers).asset_extracted_path
+  ensure
+    ENV.delete("VERSION")
+  end
+
+  test "asset volume path" do
+    ENV["VERSION"] = "12345"
+    assert_equal ".kamal/assets/volumes/app-web-12345", @config_with_roles.role(:web).asset_volume_path
+    assert_equal ".kamal/assets/volumes/app-workers-12345", @config_with_roles.role(:workers).asset_volume_path
+  ensure
+    ENV.delete("VERSION")
+  end
 end

test/configuration/ssh_test.rb

@@ -22,6 +22,9 @@ class ConfigurationSshTest < ActiveSupport::TestCase
 
     config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "log_level" => "debug" }) })
     assert_equal 0, config.ssh.options[:logger].level
+
+    config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "port" => 2222 }) })
+    assert_equal 2222, config.ssh.options[:port]
   end
 
   test "ssh options with proxy host" do

test/configuration/volume_test.rb

@@ -0,0 +1,13 @@
+require "test_helper"
+
+class ConfigurationVolumeTest < ActiveSupport::TestCase
+  test "docker args absolute" do
+    volume = Kamal::Configuration::Volume.new(host_path: "/root/foo/bar", container_path: "/assets")
+    assert_equal ["--volume", "/root/foo/bar:/assets"], volume.docker_args
+  end
+
+  test "docker args relative" do
+    volume = Kamal::Configuration::Volume.new(host_path: "foo/bar", container_path: "/assets")
+    assert_equal ["--volume", "$(pwd)/foo/bar:/assets"], volume.docker_args
+  end
+end

test/configuration_test.rb

@@ -42,6 +42,16 @@ class ConfigurationTest < ActiveSupport::TestCase
     end
   end
 
+  test "service name valid" do
+    assert Kamal::Configuration.new(@deploy.tap { _1[:service] = "hey-app1_primary" }).valid?
+  end
+
+  test "service name invalid" do
+    assert_raise(ArgumentError) do
+      Kamal::Configuration.new @deploy.tap { _1[:service] = "app.com" }
+    end
+  end
+
   test "roles" do
     assert_equal %w[ web ], @config.roles.collect(&:name)
     assert_equal %w[ web workers ], @config_with_roles.roles.collect(&:name)
@@ -58,9 +68,9 @@ class ConfigurationTest < ActiveSupport::TestCase
     assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], @config_with_roles.all_hosts
   end
 
-  test "primary web host" do
+  test "primary host" do
-    assert_equal "1.1.1.1", @config.primary_web_host
+    assert_equal "1.1.1.1", @config.primary_host
-    assert_equal "1.1.1.1", @config_with_roles.primary_web_host
+    assert_equal "1.1.1.1", @config_with_roles.primary_host
   end
 
   test "traefik hosts" do
@@ -75,7 +85,7 @@ class ConfigurationTest < ActiveSupport::TestCase
   test "version no git repo" do
     ENV.delete("VERSION")
 
-    @config.expects(:system).with("git rev-parse").returns(nil)
+    Kamal::Git.expects(:used?).returns(nil)
     error = assert_raises(RuntimeError) { @config.version}
     assert_match /no git repository found/, error.message
   end
@@ -83,16 +93,16 @@ class ConfigurationTest < ActiveSupport::TestCase
   test "version from git committed" do
     ENV.delete("VERSION")
 
-    @config.expects(:`).with("git rev-parse HEAD").returns("git-version")
+    Kamal::Git.expects(:revision).returns("git-version")
-    Kamal::Utils.expects(:uncommitted_changes).returns("")
+    Kamal::Git.expects(:uncommitted_changes).returns("")
     assert_equal "git-version", @config.version
   end
 
   test "version from git uncommitted" do
     ENV.delete("VERSION")
 
-    @config.expects(:`).with("git rev-parse HEAD").returns("git-version")
+    Kamal::Git.expects(:revision).returns("git-version")
-    Kamal::Utils.expects(:uncommitted_changes).returns("M   file\n")
+    Kamal::Git.expects(:uncommitted_changes).returns("M   file\n")
     assert_match /^git-version_uncommitted_[0-9a-f]{16}$/, @config.version
   end
 
@@ -124,12 +134,8 @@ class ConfigurationTest < ActiveSupport::TestCase
     assert_equal "app-missing", @config.service_with_version
   end
 
-  test "env with missing secret" do
+  test "healthcheck service" do
-    assert_raises(KeyError) do
+    assert_equal "healthcheck-app", @config.healthcheck_service
-      config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!({
-        env: { "secret" => [ "PASSWORD" ] }
-      }) }).ensure_env_available
-    end
   end
 
   test "valid config" do
@@ -169,6 +175,16 @@ class ConfigurationTest < ActiveSupport::TestCase
     end
   end
 
+  test "allow_empty_roles" do
+    assert_silent do
+      Kamal::Configuration.new @deploy.merge(servers: { "web" => %w[ web ], "workers" => { "hosts" => %w[ ] } }, allow_empty_roles: true)
+    end
+
+    assert_raises(ArgumentError) do
+      Kamal::Configuration.new @deploy.merge(servers: { "web" => %w[], "workers" => { "hosts" => %w[] } }, allow_empty_roles: true)
+    end
+  end
+
   test "volume_args" do
     assert_equal ["--volume", "/local/path:/container/path"], @config.volume_args
   end
@@ -210,6 +226,18 @@ class ConfigurationTest < ActiveSupport::TestCase
     end
   end
 
+  test "destination required" do
+    dest_config_file = Pathname.new(File.expand_path("fixtures/deploy_for_required_dest.yml", __dir__))
+
+    assert_raises(ArgumentError) do
+      config = Kamal::Configuration.create_from config_file: dest_config_file
+    end
+
+    assert_nothing_raised do
+      config = Kamal::Configuration.create_from config_file: dest_config_file, destination: "world"
+    end
+  end
+
   test "to_h" do
     expected_config = \
       { :roles=>["web"],
@@ -219,12 +247,12 @@ class ConfigurationTest < ActiveSupport::TestCase
         :repository=>"dhh/app",
         :absolute_image=>"dhh/app:missing",
         :service_with_version=>"app-missing",
-        :ssh_options=>{ :user=>"root", :auth_methods=>["publickey"], log_level: :fatal, keepalive: true, keepalive_interval: 30 },
+        :ssh_options=>{ :user=>"root", port: 22, log_level: :fatal, keepalive: true, keepalive_interval: 30 },
         :sshkit=>{},
         :volume_args=>["--volume", "/local/path:/container/path"],
         :builder=>{},
         :logging=>["--log-opt", "max-size=\"10m\""],
-        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }}
+        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }}
 
     assert_equal expected_config, @config.to_h
   end
@@ -265,4 +293,47 @@ class ConfigurationTest < ActiveSupport::TestCase
     SecureRandom.expects(:hex).with(16).returns("09876543211234567890098765432112")
     assert_equal "09876543211234567890098765432112", @config.run_id
   end
+
+  test "asset path" do
+    assert_nil @config.asset_path
+    assert_equal "foo", Kamal::Configuration.new(@deploy.merge!(asset_path: "foo")).asset_path
+  end
+
+  test "primary role" do
+    assert_equal "web", @config.primary_role
+
+    config = Kamal::Configuration.new(@deploy_with_roles.deep_merge({
+      servers: { "alternate_web" => { "hosts" => [ "1.1.1.4", "1.1.1.5" ] } },
+      primary_role: "alternate_web" } ))
+
+
+    assert_equal "alternate_web", config.primary_role
+    assert_equal "1.1.1.4", config.primary_host
+    assert config.role(:alternate_web).primary?
+    assert config.role(:alternate_web).running_traefik?
+  end
+
+  test "primary role missing" do
+    error = assert_raises(ArgumentError) do
+      Kamal::Configuration.new(@deploy.merge(primary_role: "bar"))
+    end
+    assert_match /bar isn't defined/, error.message
+  end
+
+  test "retain_containers" do
+    assert_equal 5, @config.retain_containers
+    config = Kamal::Configuration.new(@deploy_with_roles.merge(retain_containers: 2))
+    assert_equal 2, config.retain_containers
+
+    assert_raises(ArgumentError) { Kamal::Configuration.new(@deploy_with_roles.merge(retain_containers: 0)) }
+  end
+
+  test "push_env" do
+    assert_nil @config.push_env
+    assert_equal "all", Kamal::Configuration.new(@deploy.merge(push_env: "all")).push_env
+    assert_equal "clear", Kamal::Configuration.new(@deploy.merge(push_env: "clear")).push_env
+    assert_equal "secret", Kamal::Configuration.new(@deploy.merge(push_env: "secret")).push_env
+
+    assert_raises(ArgumentError) { Kamal::Configuration.new(@deploy_with_roles.merge(push_env: "foo")) }
+  end
 end

test/env_files_test.rb

@@ -0,0 +1,102 @@
+require "test_helper"
+
+class EnvFilesTest < ActiveSupport::TestCase
+  test "env file simple" do
+    env = {
+      "foo" => "bar",
+      "baz" => "haz"
+    }
+
+    assert_equal "foo=bar\nbaz=haz\n", Kamal::EnvFiles.new(env).clear
+    assert_equal "\n", Kamal::EnvFiles.new(env).secret
+  end
+
+  test "env file clear" do
+    env = {
+      "clear" => {
+        "foo" => "bar",
+        "baz" => "haz"
+      }
+    }
+
+    assert_equal "foo=bar\nbaz=haz\n", Kamal::EnvFiles.new(env).clear
+    assert_equal "\n", Kamal::EnvFiles.new(env).secret
+  end
+
+  test "env file empty" do
+    assert_equal "\n", Kamal::EnvFiles.new({}).secret
+    assert_equal "\n", Kamal::EnvFiles.new({}).clear
+  end
+
+  test "env file secret" do
+    ENV["PASSWORD"] = "hello"
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_equal "PASSWORD=hello\n", Kamal::EnvFiles.new(env).secret
+    assert_equal "\n", Kamal::EnvFiles.new(env).clear
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "env file secret escaped newline" do
+    ENV["PASSWORD"] = "hello\\nthere"
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_equal "PASSWORD=hello\\\\nthere\n", Kamal::EnvFiles.new(env).secret
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "env file secret newline" do
+    ENV["PASSWORD"] = "hello\nthere"
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_equal "PASSWORD=hello\\nthere\n", Kamal::EnvFiles.new(env).secret
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "env file missing secret" do
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_raises(KeyError) { Kamal::EnvFiles.new(env).secret }
+
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "env file secret and clear" do
+    ENV["PASSWORD"] = "hello"
+    env = {
+      "secret" => [ "PASSWORD" ],
+      "clear" => {
+        "foo" => "bar",
+        "baz" => "haz"
+      }
+    }
+
+    assert_equal "PASSWORD=hello\n", Kamal::EnvFiles.new(env).secret
+    assert_equal "foo=bar\nbaz=haz\n", Kamal::EnvFiles.new(env).clear
+
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "stringIO conversion" do
+    env = {
+      "foo" => "bar",
+      "baz" => "haz"
+    }
+
+    assert_equal "foo=bar\nbaz=haz\n", \
+      StringIO.new(Kamal::EnvFiles.new(env).clear).read
+  end
+end

test/fixtures/deploy_for_required_dest.world.yml

@@ -0,0 +1,5 @@
+servers:
+  - 1.1.1.1
+  - 1.1.1.2
+env:
+  REDIS_URL: redis://x/y

test/fixtures/deploy_for_required_dest.yml

@@ -0,0 +1,7 @@
+service: app
+image: dhh/app
+registry:
+  server: registry.digitalocean.com
+  username: <%= "my-user" %>
+  password: <%= "my-password" %>
+require_destination: true

test/fixtures/deploy_primary_web_role_override.yml

@@ -0,0 +1,20 @@
+service: app
+image: dhh/app
+servers:
+  web_chicago:
+    traefik: enabled
+    hosts:
+      - 1.1.1.1
+      - 1.1.1.2
+  web_tokyo:
+    traefik: enabled
+    hosts:
+      - 1.1.1.3
+      - 1.1.1.4
+env:
+  REDIS_URL: redis://x/y
+registry:
+  server: registry.digitalocean.com
+  username: user
+  password: pw
+primary_role: web_tokyo

test/fixtures/deploy_push_clear_env.yml

@@ -0,0 +1,37 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+registry:
+  username: user
+  password: pw
+push_env: clear
+
+accessories:
+  mysql:
+    image: mysql:5.7
+    host: 1.1.1.3
+    port: 3306
+    env:
+      clear:
+        MYSQL_ROOT_HOST: '%'
+      secret:
+        - MYSQL_ROOT_PASSWORD
+    files:
+      - test/fixtures/files/my.cnf:/etc/mysql/my.cnf
+    directories:
+      - data:/var/lib/mysql
+  redis:
+    image: redis:latest
+    roles:
+      - web
+    port: 6379
+    directories:
+      - data:/data
+
+readiness_delay: 0

test/fixtures/deploy_with_aliases.yml

@@ -0,0 +1,36 @@
+# helper aliases
+chicago_hosts: &chicago_hosts
+  hosts:
+    - 1.1.1.1
+    - 1.1.1.2
+tokyo_hosts: &tokyo_hosts
+  hosts:
+    - 1.1.1.3
+    - 1.1.1.4
+web_common: &web_common
+  env:
+    ROLE: "web"
+  traefik: true
+
+# actual config
+service: app
+image: dhh/app
+servers:
+  web:
+    <<: *chicago_hosts
+    <<: *web_common
+  web_tokyo:
+    <<: *tokyo_hosts
+    <<: *web_common
+  workers:
+    cmd: bin/jobs
+    <<: *chicago_hosts
+  workers_tokyo:
+    cmd: bin/jobs
+    <<: *tokyo_hosts
+env:
+  REDIS_URL: redis://x/y
+registry:
+  server: registry.digitalocean.com
+  username: user
+  password: pw

test/fixtures/deploy_with_assets.yml

@@ -0,0 +1,9 @@
+service: app
+image: dhh/app
+servers:
+  - "1.1.1.1"
+  - "1.1.1.2"
+registry:
+  username: user
+  password: pw
+asset_path: /public/assets

test/fixtures/deploy_with_remote_builder.yml

@@ -0,0 +1,41 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+registry:
+  username: user
+  password: pw
+
+accessories:
+  mysql:
+    image: mysql:5.7
+    host: 1.1.1.3
+    port: 3306
+    env:
+      clear:
+        MYSQL_ROOT_HOST: '%'
+      secret:
+        - MYSQL_ROOT_PASSWORD
+    files:
+      - test/fixtures/files/my.cnf:/etc/mysql/my.cnf
+    directories:
+      - data:/var/lib/mysql
+  redis:
+    image: redis:latest
+    roles:
+      - web
+    port: 6379
+    directories:
+      - data:/data
+
+readiness_delay: 0
+
+builder:
+  remote:
+    arch: amd64
+    host: ssh://app@1.1.1.5

test/fixtures/deploy_workers_only.yml

@@ -0,0 +1,12 @@
+service: app
+image: dhh/app
+servers:
+  workers:
+    traefik: false
+    hosts:
+      - 1.1.1.1
+      - 1.1.1.2
+primary_role: workers
+registry:
+  username: user
+  password: pw

test/git_test.rb

@@ -0,0 +1,13 @@
+require "test_helper"
+
+class GitTest < ActiveSupport::TestCase
+  test "uncommitted changes exist" do
+    Kamal::Git.expects(:`).with("git status --porcelain").returns("M   file\n")
+    assert_equal "M   file", Kamal::Git.uncommitted_changes
+  end
+
+  test "uncommitted changes do not exist" do
+    Kamal::Git.expects(:`).with("git status --porcelain").returns("")
+    assert_equal "", Kamal::Git.uncommitted_changes
+  end
+end

test/integration/app_test.rb

@@ -10,8 +10,7 @@ class AppTest < IntegrationTest
 
     kamal :app, :stop
 
-    # traefik is up and returns 404s when it can't match a route
+    assert_app_is_down
-    assert_app_not_found
 
     kamal :app, :start
 
@@ -51,7 +50,6 @@ class AppTest < IntegrationTest
 
     kamal :app, :remove
 
-    # traefik is up and returns 404s when it can't match a route
+    assert_app_is_down
-    assert_app_not_found
   end
 end

test/integration/docker/deployer/app/.kamal/hooks/post-traefik-reboot

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "Rebooted Traefik on ${KAMAL_HOSTS}"
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/post-traefik-reboot

test/integration/docker/deployer/app/.kamal/hooks/pre-traefik-reboot

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "Rebooting Traefik on ${KAMAL_HOSTS}..."
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-traefik-reboot

test/integration/docker/deployer/app/Dockerfile

@@ -5,4 +5,5 @@ COPY default.conf /etc/nginx/conf.d/default.conf
 ARG COMMIT_SHA
 RUN echo $COMMIT_SHA > /usr/share/nginx/html/version
 RUN mkdir -p /usr/share/nginx/html/versions && echo "version" > /usr/share/nginx/html/versions/$COMMIT_SHA
+RUN mkdir -p /usr/share/nginx/html/versions && echo "hidden" > /usr/share/nginx/html/versions/.hidden
 

test/integration/docker/deployer/app/config/deploy.yml

@@ -24,11 +24,13 @@ traefik:
   args:
     accesslog: true
     accesslog.format: json
-  image: registry:4443/traefik:v2.9
+  image: registry:4443/traefik:v2.10
 accessories:
   busybox:
+    service: custom-busybox
     image: registry:4443/busybox:1.36.0
     cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done'
     roles:
       - web
 stop_wait_time: 1
+push_env: clear

test/integration/docker/deployer/setup.sh

@@ -19,5 +19,9 @@ push_image_to_registry_4443() {
 
 install_kamal
 push_image_to_registry_4443 nginx 1-alpine-slim
-push_image_to_registry_4443 traefik v2.9
+push_image_to_registry_4443 traefik v2.10
 push_image_to_registry_4443 busybox 1.36.0
+
+# .ssh is on a shared volume that persists between runs. Clean it up as the
+# churn of temporary vm IPs can eventually create conflicts.
+rm -f /root/.ssh/known_hosts

test/integration/docker/shared/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.10
+FROM ubuntu:22.04
 
 WORKDIR /work
 

test/integration/docker/vm/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.10
+FROM ubuntu:22.04
 
 WORKDIR /work
 

test/integration/integration_test.rb

@@ -55,12 +55,6 @@ class IntegrationTest < ActiveSupport::TestCase
       assert_app_version(version, response) if version
     end
 
-    def assert_app_not_found
-      response = app_response
-      debug_response_code(response, "404")
-      assert_equal "404", response.code
-    end
-
     def wait_for_app_to_be_up(timeout: 20, up_count: 3)
       timeout_at = Time.now + timeout
       up_times = 0
@@ -109,7 +103,7 @@ class IntegrationTest < ActiveSupport::TestCase
       assert_equal "200", code
     end
 
-    def wait_for_healthy(timeout: 20)
+    def wait_for_healthy(timeout: 30)
       timeout_at = Time.now + timeout
       while docker_compose("ps -a | tail -n +2 | grep -v '(healthy)' | wc -l", capture: true) != "0"
         if timeout_at < Time.now

test/integration/main_test.rb

@@ -4,7 +4,10 @@ class MainTest < IntegrationTest
   test "envify, deploy, redeploy, rollback, details and audit" do
     kamal :envify
     assert_local_env_file "SECRET_TOKEN=1234"
-    assert_remote_env_file "SECRET_TOKEN=1234\nCLEAR_TOKEN=4321"
+    assert_remote_env_file "CLEAR_TOKEN=4321", :clear
+    assert_remote_env_file "SECRET_TOKEN=1234", :secret
+    remove_local_env_file
+    remove_remote_env_file :clear
 
     first_version = latest_app_version
 
@@ -13,6 +16,7 @@ class MainTest < IntegrationTest
     kamal :deploy
     assert_app_is_up version: first_version
     assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy"
+    assert_remote_env_file "CLEAR_TOKEN=4321", :clear
 
     second_version = update_app_rev
 
@@ -31,14 +35,14 @@ class MainTest < IntegrationTest
     assert_match /Traefik Host: vm2/, details
     assert_match /App Host: vm1/, details
     assert_match /App Host: vm2/, details
-    assert_match /traefik:v2.9/, details
+    assert_match /traefik:v2.10/, details
     assert_match /registry:4443\/app:#{first_version}/, details
 
     audit = kamal :audit, capture: true
     assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
 
     kamal :env, :delete
-    assert_no_remote_env_file
+    assert_no_remote_env_files
   end
 
   test "config" do
@@ -53,10 +57,10 @@ class MainTest < IntegrationTest
     assert_equal "registry:4443/app:#{version}", config[:absolute_image]
     assert_equal "app-#{version}", config[:service_with_version]
     assert_equal [], config[:volume_args]
-    assert_equal({ user: "root", auth_methods: [ "publickey" ], keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])
+    assert_equal({ user: "root", port: 22, keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])
     assert_equal({ "multiarch" => false, "args" => { "COMMIT_SHA" => version } }, config[:builder])
     assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging]
-    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord"=>"/tmp/kamal-cord", "cmd"=>"wget -qO- http://localhost > /dev/null || exit 1" }, config[:healthcheck])
+    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord"=>"/tmp/kamal-cord", "log_lines" => 50, "cmd"=>"wget -qO- http://localhost > /dev/null || exit 1" }, config[:healthcheck])
   end
 
   private
@@ -64,17 +68,28 @@ class MainTest < IntegrationTest
       assert_equal contents, deployer_exec("cat .env", capture: true)
     end
 
-    def assert_remote_env_file(contents)
+    def remove_local_env_file
-      assert_equal contents, docker_compose("exec vm1 cat /root/.kamal/env/roles/app-web.env", capture: true)
+      deployer_exec("rm .env")
     end
 
-    def assert_no_remote_env_file
+    def remove_remote_env_file(env_type)
-      assert_equal "nofile", docker_compose("exec vm1 stat /root/.kamal/env/roles/app-web.env 2> /dev/null || echo nofile", capture: true)
+      docker_compose("exec vm1 cat /root/.kamal/env/roles/app-web-#{env_type}.env")
+    end
+
+    def assert_remote_env_file(contents, env_type)
+      assert_equal contents, docker_compose("exec vm1 cat /root/.kamal/env/roles/app-web-#{env_type}.env", capture: true)
+    end
+
+    def assert_no_remote_env_files
+      assert_equal "nofile", docker_compose("exec vm1 stat /root/.kamal/env/roles/app-web-clear.env 2> /dev/null || echo nofile", capture: true)
+      assert_equal "nofile", docker_compose("exec vm1 stat /root/.kamal/env/roles/app-web-secret.env 2> /dev/null || echo nofile", capture: true)
     end
 
     def assert_accumulated_assets(*versions)
       versions.each do |version|
         assert_equal "200", Net::HTTP.get_response(URI.parse("http://localhost:12345/versions/#{version}")).code
       end
+
+      assert_equal "200", Net::HTTP.get_response(URI.parse("http://localhost:12345/versions/.hidden")).code
     end
 end

test/integration/traefik_test.rb

@@ -7,8 +7,19 @@ class TraefikTest < IntegrationTest
     kamal :traefik, :boot
     assert_traefik_running
 
-    kamal :traefik, :reboot
+    output = kamal :traefik, :reboot, capture: true
     assert_traefik_running
+    assert_hooks_ran "pre-traefik-reboot", "post-traefik-reboot"
+    assert_match /Rebooting Traefik on vm1,vm2.../, output
+    assert_match /Rebooted Traefik on vm1,vm2/, output
+
+    output = kamal :traefik, :reboot, :"--rolling", capture: true
+    assert_traefik_running
+    assert_hooks_ran "pre-traefik-reboot", "post-traefik-reboot"
+    assert_match /Rebooting Traefik on vm1.../, output
+    assert_match /Rebooted Traefik on vm1/, output
+    assert_match /Rebooting Traefik on vm2.../, output
+    assert_match /Rebooted Traefik on vm2/, output
 
     kamal :traefik, :boot
     assert_traefik_running
@@ -41,11 +52,11 @@ class TraefikTest < IntegrationTest
 
   private
     def assert_traefik_running
-      assert_match /traefik:v2.9   "\/entrypoint.sh/, traefik_details
+      assert_match /traefik:v2.10   "\/entrypoint.sh/, traefik_details
     end
 
     def assert_traefik_not_running
-      refute_match /traefik:v2.9   "\/entrypoint.sh/, traefik_details
+      refute_match /traefik:v2.10   "\/entrypoint.sh/, traefik_details
     end
 
     def traefik_details

test/utils_test.rb

@@ -11,67 +11,6 @@ class UtilsTest < ActiveSupport::TestCase
       Kamal::Utils.argumentize("--label", { foo: "bar" }, sensitive: true).last
   end
 
-  test "env file simple" do
-    env = {
-      "foo" => "bar",
-      "baz" => "haz"
-    }
-
-    assert_equal "foo=bar\nbaz=haz\n", \
-      Kamal::Utils.env_file_with_secrets(env)
-  end
-
-  test "env file clear" do
-    env = {
-      "clear" => {
-        "foo" => "bar",
-        "baz" => "haz"
-      }
-    }
-
-    assert_equal "foo=bar\nbaz=haz\n", \
-      Kamal::Utils.env_file_with_secrets(env)
-  end
-
-  test "env file secret" do
-    ENV["PASSWORD"] = "hello"
-    env = {
-      "secret" => [ "PASSWORD" ]
-    }
-
-    assert_equal "PASSWORD=hello\n", \
-      Kamal::Utils.env_file_with_secrets(env)
-  ensure
-    ENV.delete "PASSWORD"
-  end
-
-  test "env file missing secret" do
-    env = {
-      "secret" => [ "PASSWORD" ]
-    }
-
-    assert_raises(KeyError) { Kamal::Utils.env_file_with_secrets(env) }
-
-  ensure
-    ENV.delete "PASSWORD"
-  end
-
-  test "env file secret and clear" do
-    ENV["PASSWORD"] = "hello"
-    env = {
-      "secret" => [ "PASSWORD" ],
-      "clear" => {
-        "foo" => "bar",
-        "baz" => "haz"
-      }
-    }
-
-    assert_equal "PASSWORD=hello\nfoo=bar\nbaz=haz\n", \
-      Kamal::Utils.env_file_with_secrets(env)
-  ensure
-    ENV.delete "PASSWORD"
-  end
-
   test "optionize" do
     assert_equal [ "--foo", "\"bar\"", "--baz", "\"qux\"", "--quux" ], \
       Kamal::Utils.optionize({ foo: "bar", baz: "qux", quux: true })
@@ -113,14 +52,4 @@ class UtilsTest < ActiveSupport::TestCase
     assert_equal "\"https://example.com/\\$2\"",
       Kamal::Utils.escape_shell_value("https://example.com/$2")
   end
-
-  test "uncommitted changes exist" do
-    Kamal::Utils.expects(:`).with("git status --porcelain").returns("M   file\n")
-    assert_equal "M   file", Kamal::Utils.uncommitted_changes
-  end
-
-  test "uncommitted changes do not exist" do
-    Kamal::Utils.expects(:`).with("git status --porcelain").returns("")
-    assert_equal "", Kamal::Utils.uncommitted_changes
-  end
 end