Rename roles

Allow roles to be renamed without having to manually stop the old
containers.

If you have config like:

```
servers:
  jobs:
    hosts:
      - vm3
```

And you want to rename `jobs` to `workers`, you can do:

```
servers:
  workers:
    previously:
      - jobs
    hosts:
      - vm3
```

And the deployment will take care of stopping the old "jobs" containers.

Once deployed you can remove the `previously` key.

.github/workflows/ci.yml

@@ -1,10 +1,25 @@
 name: CI
-on: 
+on:
   push:
     branches:
       - main
   pull_request:
 jobs:
+  rubocop:
+    name: RuboCop
+    runs-on: ubuntu-latest
+    env:
+      BUNDLE_ONLY: rubocop
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Setup Ruby and install gems
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.3.0
+          bundler-cache: true
+      - name: Run Rubocop
+        run: bundle exec rubocop --parallel
   tests:
     strategy:
       matrix:
@@ -12,17 +27,29 @@ jobs:
           - "2.7"
           - "3.1"
           - "3.2"
+          - "3.3"
         gemfile:
           - Gemfile
+          - gemfiles/ruby_2.7.gemfile
           - gemfiles/rails_edge.gemfile
-        continue-on-error: [false]
+        exclude:
+          - ruby-version: "2.7"
+            gemfile: Gemfile
+          - ruby-version: "2.7"
+            gemfile: gemfiles/rails_edge.gemfile
+          - ruby-version: "3.1"
+            gemfile: gemfiles/ruby_2.7.gemfile
+          - ruby-version: "3.2"
+            gemfile: gemfiles/ruby_2.7.gemfile
+          - ruby-version: "3.3"
+            gemfile: gemfiles/ruby_2.7.gemfile
     name: ${{ format('Tests (Ruby {0})', matrix.ruby-version) }}
     runs-on: ubuntu-latest
-    continue-on-error: ${{ matrix.continue-on-error }}
+    continue-on-error: true
     env:
       BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install Ruby
         uses: ruby/setup-ruby@v1

.rubocop.yml

@@ -0,0 +1,2 @@
+inherit_gem:
+  rubocop-rails-omakase: rubocop.yml

Gemfile

@@ -1,4 +1,8 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 
 gemspec
+
+group :rubocop do
+  gem "rubocop-rails-omakase", require: false
+end

Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    kamal (0.16.1)
+    kamal (1.5.0)
       activesupport (>= 7.0)
+      base64 (~> 0.2)
       bcrypt_pbkdf (~> 1.0)
       concurrent-ruby (~> 1.2)
       dotenv (~> 2.8)
@@ -15,82 +16,152 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.0.4.3)
+    actionpack (7.1.2)
-      actionview (= 7.0.4.3)
+      actionview (= 7.1.2)
-      activesupport (= 7.0.4.3)
+      activesupport (= 7.1.2)
-      rack (~> 2.0, >= 2.2.0)
+      nokogiri (>= 1.8.5)
+      racc
+      rack (>= 2.2.4)
+      rack-session (>= 1.0.1)
       rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.0)
+      rails-dom-testing (~> 2.2)
-      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+      rails-html-sanitizer (~> 1.6)
-    actionview (7.0.4.3)
+    actionview (7.1.2)
-      activesupport (= 7.0.4.3)
+      activesupport (= 7.1.2)
       builder (~> 3.1)
-      erubi (~> 1.4)
+      erubi (~> 1.11)
-      rails-dom-testing (~> 2.0)
+      rails-dom-testing (~> 2.2)
-      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+      rails-html-sanitizer (~> 1.6)
-    activesupport (7.0.4.3)
+    activesupport (7.1.2)
+      base64
+      bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
+      mutex_m
       tzinfo (~> 2.0)
+    ast (2.4.2)
+    base64 (0.2.0)
     bcrypt_pbkdf (1.1.0)
+    bigdecimal (3.1.5)
     builder (3.2.4)
     concurrent-ruby (1.2.2)
+    connection_pool (2.4.1)
     crass (1.0.6)
-    debug (1.7.2)
+    debug (1.9.1)
-      irb (>= 1.5.0)
+      irb (~> 1.10)
-      reline (>= 0.3.1)
+      reline (>= 0.3.8)
     dotenv (2.8.1)
+    drb (2.2.0)
+      ruby2_keywords
     ed25519 (1.3.0)
     erubi (1.12.0)
-    i18n (1.12.0)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    io-console (0.6.0)
+    io-console (0.7.1)
-    irb (1.6.3)
+    irb (1.11.0)
-      reline (>= 0.3.0)
+      rdoc
-    loofah (2.20.0)
+      reline (>= 0.3.8)
+    json (2.7.1)
+    language_server-protocol (3.17.0.3)
+    loofah (2.22.0)
       crass (~> 1.0.2)
-      nokogiri (>= 1.5.9)
+      nokogiri (>= 1.12.0)
-    method_source (1.0.0)
+    minitest (5.20.0)
-    minitest (5.18.0)
+    mocha (2.1.0)
-    mocha (2.0.2)
       ruby2_keywords (>= 0.0.5)
+    mutex_m (0.2.0)
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
-    net-ssh (7.1.0)
+    net-ssh (7.2.1)
-    nokogiri (1.14.2-arm64-darwin)
+    nokogiri (1.16.0-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.2-x86_64-darwin)
+    nokogiri (1.16.0-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.2-x86_64-linux)
+    nokogiri (1.16.0-x86_64-linux)
       racc (~> 1.4)
-    racc (1.6.2)
+    parallel (1.24.0)
-    rack (2.2.6.4)
+    parser (3.3.0.5)
+      ast (~> 2.4.1)
+      racc
+    psych (5.1.2)
+      stringio
+    racc (1.7.3)
+    rack (3.0.8)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
     rack-test (2.1.0)
       rack (>= 1.3)
-    rails-dom-testing (2.0.3)
+    rackup (2.1.0)
-      activesupport (>= 4.2.0)
+      rack (>= 3)
+      webrick (~> 1.8)
+    rails-dom-testing (2.2.0)
+      activesupport (>= 5.0.0)
+      minitest
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.5.0)
+    rails-html-sanitizer (1.6.0)
-      loofah (~> 2.19, >= 2.19.1)
+      loofah (~> 2.21)
-    railties (7.0.4.3)
+      nokogiri (~> 1.14)
-      actionpack (= 7.0.4.3)
+    railties (7.1.2)
-      activesupport (= 7.0.4.3)
+      actionpack (= 7.1.2)
-      method_source
+      activesupport (= 7.1.2)
+      irb
+      rackup (>= 1.0.0)
       rake (>= 12.2)
-      thor (~> 1.0)
+      thor (~> 1.0, >= 1.2.2)
-      zeitwerk (~> 2.5)
+      zeitwerk (~> 2.6)
-    rake (13.0.6)
+    rainbow (3.1.1)
-    reline (0.3.3)
+    rake (13.1.0)
+    rdoc (6.6.2)
+      psych (>= 4.0.0)
+    regexp_parser (2.9.0)
+    reline (0.4.2)
       io-console (~> 0.5)
+    rexml (3.2.6)
+    rubocop (1.62.1)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.31.2)
+      parser (>= 3.3.0.4)
+    rubocop-minitest (0.35.0)
+      rubocop (>= 1.61, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-performance (1.20.2)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
+    rubocop-rails (2.24.0)
+      activesupport (>= 4.2.0)
+      rack (>= 1.1)
+      rubocop (>= 1.33.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-rails-omakase (1.0.0)
+      rubocop
+      rubocop-minitest
+      rubocop-performance
+      rubocop-rails
+    ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
-    sshkit (1.21.4)
+    sshkit (1.21.7)
+      mutex_m
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
-    thor (1.2.1)
+    stringio (3.1.0)
+    thor (1.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.7)
+    unicode-display_width (2.5.0)
+    webrick (1.8.1)
+    zeitwerk (2.6.12)
 
 PLATFORMS
   arm64-darwin
@@ -102,6 +173,7 @@ DEPENDENCIES
   kamal!
   mocha
   railties
+  rubocop-rails-omakase
 
 BUNDLED WITH
    2.4.3

README.md

@@ -1,6 +1,6 @@
 # Kamal: Deploy web apps anywhere
 
-From bare metal to cloud VMs using Docker, deploy web apps anywhere with zero downtime. Kamal uses the dynamic reverse-proxy Traefik to hold requests, while the new app container is started and the old one is stopped — working seamlessly across multiple hosts, using SSHKit to execute commands. Originally built for Rails apps, Kamal will work with any type of web app that can be containerized with Docker.
+From bare metal to cloud VMs, deploy web apps anywhere with zero downtime. Kamal has the dynamic reverse-proxy Traefik hold requests while a new app container is started and the old one is stopped. Works seamlessly across multiple hosts, using SSHKit to execute commands. Originally built for Rails apps, Kamal will work with any type of web app that can be containerized with Docker.
 
 ➡️ See [kamal-deploy.org](https://kamal-deploy.org) for documentation on [installation](https://kamal-deploy.org/docs/installation), [configuration](https://kamal-deploy.org/docs/configuration), and [commands](https://kamal-deploy.org/docs/commands).
 

gemfiles/ruby_2.7.gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+gemspec path: "../"
+
+gem "nokogiri", "~> 1.15.0"

kamal.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "ed25519", "~> 1.2"
   spec.add_dependency "bcrypt_pbkdf", "~> 1.0"
   spec.add_dependency "concurrent-ruby", "~> 1.2"
+  spec.add_dependency "base64", "~> 0.2"
 
   spec.add_development_dependency "debug"
   spec.add_development_dependency "mocha"

lib/kamal.rb

@@ -5,6 +5,6 @@ require "active_support"
 require "zeitwerk"
 
 loader = Zeitwerk::Loader.for_gem
-loader.ignore("#{__dir__}/kamal/sshkit_with_ext.rb")
+loader.ignore(File.join(__dir__, "kamal", "sshkit_with_ext.rb"))
 loader.setup
 loader.eager_load # We need all commands loaded.

lib/kamal/cli/accessory.rb

@@ -5,11 +5,11 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
       if name == "all"
         KAMAL.accessory_names.each { |accessory_name| boot(accessory_name) }
       else
-        with_accessory(name) do |accessory|
+        with_accessory(name) do |accessory, hosts|
           directories(name)
           upload(name)
 
-          on(accessory.hosts) do
+          on(hosts) do
             execute *KAMAL.registry.login if login
             execute *KAMAL.auditor.record("Booted #{name} accessory"), verbosity: :debug
             execute *accessory.run
@@ -22,8 +22,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "upload [NAME]", "Upload accessory files to host", hide: true
   def upload(name)
     mutating do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           accessory.files.each do |(local, remote)|
             accessory.ensure_local_file_present(local)
 
@@ -39,8 +39,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "directories [NAME]", "Create accessory directories on host", hide: true
   def directories(name)
     mutating do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           accessory.directories.keys.each do |host_path|
             execute *accessory.make_directory(host_path)
           end
@@ -49,17 +49,21 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     end
   end
 
-  desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container)"
+  desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container; use NAME=all to boot all accessories)"
   def reboot(name)
     mutating do
-      with_accessory(name) do |accessory|
+      if name == "all"
-        on(accessory.hosts) do
+        KAMAL.accessory_names.each { |accessory_name| reboot(accessory_name) }
-          execute *KAMAL.registry.login
+      else
-        end
+        with_accessory(name) do |accessory, hosts|
+          on(hosts) do
+            execute *KAMAL.registry.login
+          end
 
-        stop(name)
+          stop(name)
-        remove_container(name)
+          remove_container(name)
-        boot(name, login: false)
+          boot(name, login: false)
+        end
       end
     end
   end
@@ -67,8 +71,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "start [NAME]", "Start existing accessory container on host"
   def start(name)
     mutating do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Started #{name} accessory"), verbosity: :debug
           execute *accessory.start
         end
@@ -79,8 +83,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "stop [NAME]", "Stop existing accessory container on host"
   def stop(name)
     mutating do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Stopped #{name} accessory"), verbosity: :debug
           execute *accessory.stop, raise_on_non_zero_exit: false
         end
@@ -103,8 +107,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     if name == "all"
       KAMAL.accessory_names.each { |accessory_name| details(accessory_name) }
     else
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) { puts capture_with_info(*accessory.info) }
+        on(hosts) { puts capture_with_info(*accessory.info) }
       end
     end
   end
@@ -113,7 +117,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
   option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
   def exec(name, cmd)
-    with_accessory(name) do |accessory|
+    with_accessory(name) do |accessory, hosts|
       case
       when options[:interactive] && options[:reuse]
         say "Launching interactive command with via SSH from existing container...", :magenta
@@ -125,14 +129,14 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
       when options[:reuse]
         say "Launching command from existing container...", :magenta
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
           capture_with_info(*accessory.execute_in_existing_container(cmd))
         end
 
       else
         say "Launching command from new container...", :magenta
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
           capture_with_info(*accessory.execute_in_new_container(cmd))
         end
@@ -146,12 +150,12 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
   option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
   def logs(name)
-    with_accessory(name) do |accessory|
+    with_accessory(name) do |accessory, hosts|
       grep = options[:grep]
 
       if options[:follow]
         run_locally do
-          info "Following logs on #{accessory.hosts}..."
+          info "Following logs on #{hosts}..."
           info accessory.follow_logs(grep: grep)
           exec accessory.follow_logs(grep: grep)
         end
@@ -159,7 +163,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
         since = options[:since]
         lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
-        on(accessory.hosts) do
+        on(hosts) do
           puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep))
         end
       end
@@ -173,7 +177,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
       if name == "all"
         KAMAL.accessory_names.each { |accessory_name| remove(accessory_name) }
       else
-        if options[:confirmed] || ask("This will remove all containers, images and data directories for #{name}. Are you sure?", limited_to: %w( y N ), default: "N") == "y"
+        confirming "This will remove all containers, images and data directories for #{name}. Are you sure?" do
           with_accessory(name) do
             stop(name)
             remove_container(name)
@@ -188,8 +192,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "remove_container [NAME]", "Remove accessory container from host", hide: true
   def remove_container(name)
     mutating do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Remove #{name} accessory container"), verbosity: :debug
           execute *accessory.remove_container
         end
@@ -200,8 +204,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "remove_image [NAME]", "Remove accessory image from host", hide: true
   def remove_image(name)
     mutating do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Removed #{name} accessory image"), verbosity: :debug
           execute *accessory.remove_image
         end
@@ -212,8 +216,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "remove_service_directory [NAME]", "Remove accessory directory used for uploaded files and data directories from host", hide: true
   def remove_service_directory(name)
     mutating do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *accessory.remove_service_directory
         end
       end
@@ -222,8 +226,9 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   private
     def with_accessory(name)
-      if accessory = KAMAL.accessory(name)
+      if KAMAL.config.accessory(name)
-        yield accessory
+        accessory = KAMAL.accessory(name)
+        yield accessory, accessory_hosts(accessory)
       else
         error_on_missing_accessory(name)
       end
@@ -236,4 +241,12 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
         "No accessory by the name of '#{name}'" +
         (options ? " (options: #{options.to_sentence})" : "")
     end
+
+    def accessory_hosts(accessory)
+      if KAMAL.specific_hosts&.any?
+        KAMAL.specific_hosts & accessory.hosts
+      else
+        accessory.hosts
+      end
+    end
 end

lib/kamal/cli/app.rb

@@ -7,35 +7,23 @@ class Kamal::Cli::App < Kamal::Cli::Base
         using_version(version_or_latest) do |version|
           say "Start container with version #{version} using a #{KAMAL.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
 
+          # Assets are prepared in a separate step to ensure they are on all hosts before booting
           on(KAMAL.hosts) do
-            execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
+            KAMAL.roles_on(host).each do |role|
-            execute *KAMAL.app.tag_current_as_latest
+              Kamal::Cli::App::PrepareAssets.new(host, role, self).run
+            end
           end
 
           on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
-            roles = KAMAL.roles_on(host)
+            KAMAL.roles_on(host).each do |role|
-
+              Kamal::Cli::App::Boot.new(host, role, version, self).run
-            roles.each do |role|
-              app = KAMAL.app(role: role)
-              auditor = KAMAL.auditor(role: role)
-
-              if capture_with_info(*app.container_id_for_version(version, only_running: true), raise_on_non_zero_exit: false).present?
-                tmp_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
-                info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
-                execute *auditor.record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
-                execute *app.rename_container(version: version, new_version: tmp_version)
-              end
-
-              execute *auditor.record("Booted app version #{version}"), verbosity: :debug
-
-              old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-              execute *app.start_or_run(hostname: "#{host}-#{SecureRandom.hex(6)}")
-
-              Kamal::Utils::HealthcheckPoller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
-
-              execute *app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
             end
           end
+
+          on(KAMAL.hosts) do |host|
+            execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
+            execute *KAMAL.app.tag_latest_image
+          end
         end
       end
     end
@@ -84,20 +72,24 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "exec [CMD]", "Execute a custom command on servers (use --help to show options)"
   option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
   option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
+  option :env, aliases: "-e", type: :hash, desc: "Set environment variables for the command"
   def exec(cmd)
+    env = options[:env]
     case
     when options[:interactive] && options[:reuse]
       say "Get current version of running container...", :magenta unless options[:version]
       using_version(options[:version] || current_running_version) do |version|
         say "Launching interactive command with version #{version} via SSH from existing container on #{KAMAL.primary_host}...", :magenta
-        run_locally { exec KAMAL.app(role: "web").execute_in_existing_container_over_ssh(cmd, host: KAMAL.primary_host) }
+        run_locally { exec KAMAL.app(role: KAMAL.primary_role).execute_in_existing_container_over_ssh(cmd, host: KAMAL.primary_host, env: env) }
       end
 
     when options[:interactive]
       say "Get most recent version available as an image...", :magenta unless options[:version]
       using_version(version_or_latest) do |version|
         say "Launching interactive command with version #{version} via SSH from new container on #{KAMAL.primary_host}...", :magenta
-        run_locally { exec KAMAL.app(role: KAMAL.roles_on(KAMAL.primary_host).first).execute_in_new_container_over_ssh(cmd, host: KAMAL.primary_host) }
+        run_locally do
+          exec KAMAL.app(role: KAMAL.primary_role).execute_in_new_container_over_ssh(cmd, host: KAMAL.primary_host, env: env)
+        end
       end
 
     when options[:reuse]
@@ -110,7 +102,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
           roles.each do |role|
             execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}", role: role), verbosity: :debug
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_existing_container(cmd))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_existing_container(cmd, env: env))
           end
         end
       end
@@ -120,8 +112,12 @@ class Kamal::Cli::App < Kamal::Cli::Base
       using_version(version_or_latest) do |version|
         say "Launching command with version #{version} from new container...", :magenta
         on(KAMAL.hosts) do |host|
-          execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
+          roles = KAMAL.roles_on(host)
-          puts_by_host host, capture_with_info(*KAMAL.app.execute_in_new_container(cmd))
+
+          roles.each do |role|
+            execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_new_container(cmd, env: env))
+          end
         end
       end
     end
@@ -144,7 +140,10 @@ class Kamal::Cli::App < Kamal::Cli::Base
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
-          cli.send(:stale_versions, host: host, role: role).each do |version|
+          versions = capture_with_info(*KAMAL.app(role: role).list_versions, raise_on_non_zero_exit: false).split("\n")
+          versions -= [ capture_with_info(*KAMAL.app(role: role).current_running_version, raise_on_non_zero_exit: false).strip ]
+
+          versions.each do |version|
             if stop
               puts_by_host host, "Stopping stale container for role #{role} with version #{version}"
               execute *KAMAL.app(role: role).stop(version: version), raise_on_non_zero_exit: false
@@ -171,19 +170,20 @@ class Kamal::Cli::App < Kamal::Cli::Base
     # FIXME: Catch when app containers aren't running
 
     grep = options[:grep]
-
+    since = options[:since]
     if options[:follow]
+      lines = options[:lines].presence || ((since || grep) ? nil : 10) # Default to 10 lines if since or grep isn't set
+
       run_locally do
         info "Following logs on #{KAMAL.primary_host}..."
 
-        KAMAL.specific_roles ||= ["web"]
+        KAMAL.specific_roles ||= [ "web" ]
         role = KAMAL.roles_on(KAMAL.primary_host).first
 
-        info KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, grep: grep)
+        info KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
-        exec KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, grep: grep)
+        exec KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
       end
     else
-      since = options[:since]
       lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
       on(KAMAL.hosts) do |host|
@@ -279,18 +279,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
       version.presence
     end
 
-    def stale_versions(host:, role:)
-      versions = nil
-      on(host) do
-        versions = \
-          capture_with_info(*KAMAL.app(role: role).list_versions, raise_on_non_zero_exit: false)
-          .split("\n")
-          .drop(1)
-      end
-      versions
-    end
-
     def version_or_latest
-      options[:version] || "latest"
+      options[:version] || KAMAL.config.latest_tag
     end
 end

lib/kamal/cli/app/boot.rb

@@ -0,0 +1,73 @@
+class Kamal::Cli::App::Boot
+  attr_reader :host, :role, :version, :sshkit
+  delegate :execute, :capture_with_info, :info, to: :sshkit
+  delegate :uses_cord?, :assets?, to: :role
+
+  def initialize(host, role, version, sshkit)
+    @host = host
+    @role = role
+    @version = version
+    @sshkit = sshkit
+  end
+
+  def run
+    old_version, old_app = old_version_renamed_if_clashing
+
+    start_new_version
+
+    if old_version
+      stop_old_version(old_version, old_app)
+    end
+  end
+
+  private
+    def app
+      @app ||= KAMAL.app(role: role)
+    end
+
+    def auditor
+      @auditor = KAMAL.auditor(role: role)
+    end
+
+    def audit(message)
+      execute *auditor.record(message), verbosity: :debug
+    end
+
+    def old_version_renamed_if_clashing
+      if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
+        renamed_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
+        info "Renaming container #{version} to #{renamed_version} as already deployed on #{host}"
+        audit("Renaming container #{version} to #{renamed_version}")
+        execute *app.rename_container(version: version, new_version: renamed_version)
+      end
+
+      [ role, *role.previous_roles ].each do |old_role|
+        old_app = KAMAL.app(role: old_role)
+        old_version = capture_with_info(*old_app.current_running_version, raise_on_non_zero_exit: false).strip.presence
+        return [ old_version, old_app ] if old_version
+      end
+
+      nil
+    end
+
+    def start_new_version
+      audit "Booted app version #{version}"
+      execute *app.tie_cord(role.cord_host_file) if uses_cord?
+      execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
+      Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+    end
+
+    def stop_old_version(version, app)
+      if uses_cord?
+        cord = capture_with_info(*app.cord(version: version), raise_on_non_zero_exit: false).strip
+        if cord.present?
+          execute *app.cut_cord(cord)
+          Kamal::Cli::Healthcheck::Poller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+        end
+      end
+
+      execute *app.stop(version: version), raise_on_non_zero_exit: false
+
+      execute *app.clean_up_assets if assets?
+    end
+end

lib/kamal/cli/app/prepare_assets.rb

@@ -0,0 +1,24 @@
+class Kamal::Cli::App::PrepareAssets
+  attr_reader :host, :role, :sshkit
+  delegate :execute, :capture_with_info, :info, to: :sshkit
+  delegate :assets?, to: :role
+
+  def initialize(host, role, sshkit)
+    @host = host
+    @role = role
+    @sshkit = sshkit
+  end
+
+  def run
+    if assets?
+      execute *app.extract_assets
+      old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+      execute *app.sync_asset_volumes(old_version: old_version)
+    end
+  end
+
+  private
+    def app
+      @app ||= KAMAL.app(role: role)
+    end
+end

lib/kamal/cli/base.rb

@@ -14,8 +14,8 @@ module Kamal::Cli
     class_option :version, desc: "Run commands against a specific app version"
 
     class_option :primary, type: :boolean, aliases: "-p", desc: "Run commands only on primary host instead of all"
-    class_option :hosts, aliases: "-h", desc: "Run commands on these hosts instead of all (separate by comma)"
+    class_option :hosts, aliases: "-h", desc: "Run commands on these hosts instead of all (separate by comma, supports wildcards with *)"
-    class_option :roles, aliases: "-r", desc: "Run commands on these roles instead of all (separate by comma)"
+    class_option :roles, aliases: "-r", desc: "Run commands on these roles instead of all (separate by comma, supports wildcards with *)"
 
     class_option :config_file, aliases: "-c", default: "config/deploy.yml", desc: "Path to config file"
     class_option :destination, aliases: "-d", desc: "Specify destination to be used for config file (staging -> deploy.staging.yml)"
@@ -24,6 +24,7 @@ module Kamal::Cli
 
     def initialize(*)
       super
+      @original_env = ENV.to_h.dup
       load_envs
       initialize_commander(options_with_subcommand_class_options)
     end
@@ -37,6 +38,12 @@ module Kamal::Cli
         end
       end
 
+      def reload_envs
+        ENV.clear
+        ENV.update(@original_env)
+        load_envs
+      end
+
       def options_with_subcommand_class_options
         options.merge(@_initializer.last[:class_options] || {})
       end
@@ -66,7 +73,7 @@ module Kamal::Cli
       def print_runtime
         started_at = Time.now
         yield
-        return Time.now - started_at
+        Time.now - started_at
       ensure
         runtime = Time.now - started_at
         puts "  Finished all in #{sprintf("%.1f seconds", runtime)}"
@@ -75,10 +82,10 @@ module Kamal::Cli
       def mutating
         return yield if KAMAL.holding_lock?
 
-        KAMAL.config.ensure_env_available
-
         run_hook "pre-connect"
 
+        ensure_run_and_locks_directory
+
         acquire_lock
 
         begin
@@ -96,6 +103,16 @@ module Kamal::Cli
         release_lock
       end
 
+      def confirming(question)
+        return yield if options[:confirmed]
+
+        if ask(question, limited_to: %w[ y N ], default: "N") == "y"
+          yield
+        else
+          say "Aborted", :red
+        end
+      end
+
       def acquire_lock
         raise_if_locked do
           say "Acquiring the deploy lock...", :magenta
@@ -116,8 +133,9 @@ module Kamal::Cli
         yield
       rescue SSHKit::Runner::ExecuteError => e
         if e.message =~ /cannot create directory/
+          say "Deploy lock already in place!", :red
           on(KAMAL.primary_host) { puts capture_with_debug(*KAMAL.lock.status) }
-          raise LockError, "Deploy lock found"
+          raise LockError, "Deploy lock found. Run 'kamal lock help' for more information"
         else
           raise e
         end
@@ -139,9 +157,9 @@ module Kamal::Cli
 
           say "Running the #{hook} hook...", :magenta
           run_locally do
-            KAMAL.with_verbosity(:debug) { execute *KAMAL.hook.run(hook, **details, **extra_details) }
+            execute *KAMAL.hook.run(hook, **details, **extra_details)
-          rescue SSHKit::Command::Failed
+          rescue SSHKit::Command::Failed => e
-            raise HookError.new("Hook `#{hook}` failed")
+            raise HookError.new("Hook `#{hook}` failed:\n#{e.message}")
           end
         end
       end
@@ -167,5 +185,15 @@ module Kamal::Cli
       def first_invocation
         instance_variable_get("@_invocations").first
       end
-    end
+
+      def ensure_run_and_locks_directory
+        on(KAMAL.hosts) do
+          execute(*KAMAL.server.ensure_run_directory)
+        end
+
+        on(KAMAL.primary_host) do
+          execute(*KAMAL.lock.ensure_locks_directory)
+        end
+      end
+  end
 end

lib/kamal/cli/build.rb

@@ -1,3 +1,5 @@
+require "uri"
+
 class Kamal::Cli::Build < Kamal::Cli::Base
   class BuildError < StandardError; end
 
@@ -17,7 +19,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base
       verify_local_dependencies
       run_hook "pre-build"
 
-      if (uncommitted_changes = Kamal::Utils.uncommitted_changes).present?
+      if (uncommitted_changes = Kamal::Git.uncommitted_changes).present?
         say "The following paths have uncommitted changes:\n #{uncommitted_changes}", :yellow
       end
 
@@ -48,6 +50,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base
         execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
         execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
         execute *KAMAL.builder.pull
+        execute *KAMAL.builder.validate_image
       end
     end
   end
@@ -55,6 +58,10 @@ class Kamal::Cli::Build < Kamal::Cli::Base
   desc "create", "Create a build setup"
   def create
     mutating do
+      if (remote_host = KAMAL.config.builder.remote_host)
+        connect_to_remote_host(remote_host)
+      end
+
       run_locally do
         begin
           debug "Using builder: #{KAMAL.builder.name}"
@@ -103,4 +110,14 @@ class Kamal::Cli::Build < Kamal::Cli::Base
         end
       end
     end
+
+    def connect_to_remote_host(remote_host)
+      remote_uri = URI.parse(remote_host)
+      if remote_uri.scheme == "ssh"
+        options = { user: remote_uri.user, port: remote_uri.port }.compact
+        on(remote_uri.host, options) do
+          execute "true"
+        end
+      end
+    end
 end

lib/kamal/cli/env.rb

@@ -0,0 +1,54 @@
+require "tempfile"
+
+class Kamal::Cli::Env < Kamal::Cli::Base
+  desc "push", "Push the env file to the remote hosts"
+  def push
+    mutating do
+      on(KAMAL.hosts) do
+        execute *KAMAL.auditor.record("Pushed env files"), verbosity: :debug
+
+        KAMAL.roles_on(host).each do |role|
+          execute *KAMAL.app(role: role).make_env_directory
+          upload! role.env.secrets_io, role.env.secrets_file, mode: 400
+        end
+      end
+
+      on(KAMAL.traefik_hosts) do
+        execute *KAMAL.traefik.make_env_directory
+        upload! KAMAL.traefik.env.secrets_io, KAMAL.traefik.env.secrets_file, mode: 400
+      end
+
+      on(KAMAL.accessory_hosts) do
+        KAMAL.accessories_on(host).each do |accessory|
+          accessory_config = KAMAL.config.accessory(accessory)
+          execute *KAMAL.accessory(accessory).make_env_directory
+          upload! accessory_config.env.secrets_io, accessory_config.env.secrets_file, mode: 400
+        end
+      end
+    end
+  end
+
+  desc "delete", "Delete the env file from the remote hosts"
+  def delete
+    mutating do
+      on(KAMAL.hosts) do
+        execute *KAMAL.auditor.record("Deleted env files"), verbosity: :debug
+
+        KAMAL.roles_on(host).each do |role|
+          execute *KAMAL.app(role: role).remove_env_file
+        end
+      end
+
+      on(KAMAL.traefik_hosts) do
+        execute *KAMAL.traefik.remove_env_file
+      end
+
+      on(KAMAL.accessory_hosts) do
+        KAMAL.accessories_on(host).each do |accessory|
+          accessory_config = KAMAL.config.accessory(accessory)
+          execute *KAMAL.accessory(accessory).remove_env_file
+        end
+      end
+    end
+  end
+end

lib/kamal/cli/healthcheck.rb

@@ -3,11 +3,12 @@ class Kamal::Cli::Healthcheck < Kamal::Cli::Base
 
   desc "perform", "Health check current app version"
   def perform
+    raise "The primary host is not configured to run Traefik" unless KAMAL.config.role(KAMAL.config.primary_role).running_traefik?
     on(KAMAL.primary_host) do
       begin
         execute *KAMAL.healthcheck.run
-        Kamal::Utils::HealthcheckPoller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
+        Poller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
-      rescue Kamal::Utils::HealthcheckPoller::HealthcheckError => e
+      rescue Poller::HealthcheckError => e
         error capture_with_info(*KAMAL.healthcheck.logs)
         error capture_with_pretty_json(*KAMAL.healthcheck.container_health_log)
         raise

lib/kamal/cli/healthcheck/poller.rb

@@ -0,0 +1,64 @@
+module Kamal::Cli::Healthcheck::Poller
+  extend self
+
+  TRAEFIK_UPDATE_DELAY = 5
+
+  class HealthcheckError < StandardError; end
+
+  def wait_for_healthy(pause_after_ready: false, &block)
+    attempt = 1
+    max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+    begin
+      case status = block.call
+      when "healthy"
+        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
+      when "running" # No health check configured
+        sleep KAMAL.config.readiness_delay if pause_after_ready
+      else
+        raise HealthcheckError, "container not ready (#{status})"
+      end
+    rescue HealthcheckError => e
+      if attempt <= max_attempts
+        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+        sleep attempt
+        attempt += 1
+        retry
+      else
+        raise
+      end
+    end
+
+    info "Container is healthy!"
+  end
+
+  def wait_for_unhealthy(pause_after_ready: false, &block)
+    attempt = 1
+    max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+    begin
+      case status = block.call
+      when "unhealthy"
+        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
+      else
+        raise HealthcheckError, "container not unhealthy (#{status})"
+      end
+    rescue HealthcheckError => e
+      if attempt <= max_attempts
+        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+        sleep attempt
+        attempt += 1
+        retry
+      else
+        raise
+      end
+    end
+
+    info "Container is unhealthy!"
+  end
+
+  private
+    def info(message)
+      SSHKit.config.output.info(message)
+    end
+end

lib/kamal/cli/lock.rb

@@ -2,7 +2,10 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   desc "status", "Report lock status"
   def status
     handle_missing_lock do
-      on(KAMAL.primary_host) { puts capture_with_debug(*KAMAL.lock.status) }
+      on(KAMAL.primary_host) do
+        execute *KAMAL.server.ensure_run_directory
+        puts capture_with_debug(*KAMAL.lock.status)
+      end
     end
   end
 
@@ -11,7 +14,10 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   def acquire
     message = options[:message]
     raise_if_locked do
-      on(KAMAL.primary_host) { execute *KAMAL.lock.acquire(message, KAMAL.config.version), verbosity: :debug }
+      on(KAMAL.primary_host) do
+        execute *KAMAL.server.ensure_run_directory
+        execute *KAMAL.lock.acquire(message, KAMAL.config.version), verbosity: :debug
+      end
       say "Acquired the deploy lock"
     end
   end
@@ -19,7 +25,10 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   desc "release", "Release the deploy lock"
   def release
     handle_missing_lock do
-      on(KAMAL.primary_host) { execute *KAMAL.lock.release, verbosity: :debug }
+      on(KAMAL.primary_host) do
+        execute *KAMAL.server.ensure_run_directory
+        execute *KAMAL.lock.release, verbosity: :debug
+      end
       say "Released the deploy lock"
     end
   end

lib/kamal/cli/main.rb

@@ -1,10 +1,18 @@
 class Kamal::Cli::Main < Kamal::Cli::Base
-  desc "setup", "Setup all accessories and deploy app to servers"
+  desc "setup", "Setup all accessories, push the env, and deploy app to servers"
+  option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def setup
     print_runtime do
       mutating do
-        invoke "kamal:cli:server:bootstrap"
+        invoke_options = deploy_options
-        invoke "kamal:cli:accessory:boot", [ "all" ]
+
+        say "Ensure Docker is installed...", :magenta
+        invoke "kamal:cli:server:bootstrap", [], invoke_options
+
+        say "Push env files...", :magenta
+        invoke "kamal:cli:env:push", [], invoke_options
+
+        invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options
         deploy
       end
     end
@@ -33,11 +41,13 @@ class Kamal::Cli::Main < Kamal::Cli::Base
         say "Ensure Traefik is running...", :magenta
         invoke "kamal:cli:traefik:boot", [], invoke_options
 
-        say "Ensure app can pass healthcheck...", :magenta
+        if KAMAL.config.role(KAMAL.config.primary_role).running_traefik?
-        invoke "kamal:cli:healthcheck:perform", [], invoke_options
+          say "Ensure app can pass healthcheck...", :magenta
+          invoke "kamal:cli:healthcheck:perform", [], invoke_options
+        end
 
         say "Detect stale containers...", :magenta
-        invoke "kamal:cli:app:stale_containers", [], invoke_options
+        invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
 
         invoke "kamal:cli:app:boot", [], invoke_options
 
@@ -70,7 +80,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
         invoke "kamal:cli:healthcheck:perform", [], invoke_options
 
         say "Detect stale containers...", :magenta
-        invoke "kamal:cli:app:stale_containers", [], invoke_options
+        invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
 
         invoke "kamal:cli:app:boot", [], invoke_options
       end
@@ -165,6 +175,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   end
 
   desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)"
+  option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip .env file push"
   def envify
     if destination = options[:destination]
       env_template_path = ".env.#{destination}.erb"
@@ -174,14 +185,19 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       env_path          = ".env"
     end
 
-    File.write(env_path, ERB.new(File.read(env_template_path)).result, perm: 0600)
+    File.write(env_path, ERB.new(File.read(env_template_path), trim_mode: "-").result, perm: 0600)
+
+    unless options[:skip_push]
+      reload_envs
+      invoke "kamal:cli:env:push", options
+    end
   end
 
   desc "remove", "Remove Traefik, app, accessories, and registry session from servers"
   option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def remove
     mutating do
-      if options[:confirmed] || ask("This will remove all containers and images. Are you sure?", limited_to: %w( y N ), default: "N") == "y"
+      confirming "This will remove all containers and images. Are you sure?" do
         invoke "kamal:cli:traefik:remove", [], options.without(:confirmed)
         invoke "kamal:cli:app:remove", [], options.without(:confirmed)
         invoke "kamal:cli:accessory:remove", [ "all" ], options
@@ -204,6 +220,9 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   desc "build", "Build application image"
   subcommand "build", Kamal::Cli::Build
 
+  desc "env", "Manage environment files"
+  subcommand "env", Kamal::Cli::Env
+
   desc "healthcheck", "Healthcheck application"
   subcommand "healthcheck", Kamal::Cli::Healthcheck
 

lib/kamal/cli/prune.rb

@@ -7,7 +7,7 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
     end
   end
 
-  desc "images", "Prune dangling images"
+  desc "images", "Prune unused images"
   def images
     mutating do
       on(KAMAL.hosts) do
@@ -18,12 +18,17 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
     end
   end
 
-  desc "containers", "Prune all stopped containers, except the last 5"
+  desc "containers", "Prune all stopped containers, except the last n (default 5)"
+  option :retain, type: :numeric, default: nil, desc: "Number of containers to retain"
   def containers
+    retain = options.fetch(:retain, KAMAL.config.retain_containers)
+    raise "retain must be at least 1" if retain < 1
+
     mutating do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug
-        execute *KAMAL.prune.containers
+        execute *KAMAL.prune.app_containers(retain: retain)
+        execute *KAMAL.prune.healthcheck_containers
       end
     end
   end

lib/kamal/cli/server.rb

@@ -12,10 +12,14 @@ class Kamal::Cli::Server < Kamal::Cli::Base
           missing << host
         end
       end
+
+      execute(*KAMAL.server.ensure_run_directory)
     end
 
     if missing.any?
-      raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/"
+      raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and either `wget` or `curl`. Install Docker manually: https://docs.docker.com/engine/install/"
     end
+
+    run_hook "docker-setup"
   end
 end

lib/kamal/cli/templates/deploy.yml

@@ -19,6 +19,7 @@ registry:
     - KAMAL_REGISTRY_PASSWORD
 
 # Inject ENV variables into containers (secrets come from .env).
+# Remember to run `kamal env push` after making changes!
 # env:
 #   clear:
 #     DB_HOST: 192.168.0.2
@@ -52,7 +53,7 @@ registry:
 #         - MYSQL_ROOT_PASSWORD
 #     files:
 #       - config/mysql/production.cnf:/etc/mysql/my.cnf
-#       - db/production.sql.erb:/docker-entrypoint-initdb.d/setup.sql
+#       - db/production.sql:/docker-entrypoint-initdb.d/setup.sql
 #     directories:
 #       - data:/var/lib/mysql
 #   redis:
@@ -62,7 +63,7 @@ registry:
 #     directories:
 #       - data:/data
 
-# Configure custom arguments for Traefik
+# Configure custom arguments for Traefik. Be sure to reboot traefik when you modify it.
 # traefik:
 #   args:
 #     accesslog: true
@@ -72,3 +73,29 @@ registry:
 # healthcheck:
 #   path: /healthz
 #   port: 4000
+
+# Bridge fingerprinted assets, like JS and CSS, between versions to avoid
+# hitting 404 on in-flight requests. Combines all files from new and old
+# version inside the asset_path.
+#
+# If your app is using the Sprockets gem, ensure it sets `config.assets.manifest`.
+# See https://github.com/basecamp/kamal/issues/626 for details
+#
+# asset_path: /rails/public/assets
+
+# Configure rolling deploys by setting a wait time between batches of restarts.
+# boot:
+#   limit: 10 # Can also specify as a percentage of total hosts, such as "25%"
+#   wait: 2
+
+# Configure the role used to determine the primary_host. This host takes
+# deploy locks, runs health checks during the deploy, and follow logs, etc.
+#
+# Caution: there's no support for role renaming yet, so be careful to cleanup
+#          the previous role on the deployed hosts.
+# primary_role: web
+
+# Controls if we abort when see a role with no hosts. Disabling this may be
+# useful for more complex deploy configurations.
+#
+# allow_empty_roles: false

lib/kamal/cli/templates/sample_hooks/docker-setup.sample

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+# A sample docker-setup hook
+#
+# Sets up a Docker network which can then be used by the application’s containers
+
+ssh user@example.com docker network create kamal

lib/kamal/cli/templates/sample_hooks/post-traefik-reboot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooted Traefik on $KAMAL_HOSTS"

lib/kamal/cli/templates/sample_hooks/pre-build.sample

@@ -32,7 +32,7 @@ fi
 current_branch=$(git branch --show-current)
 
 if [ -z "$current_branch" ]; then
-  echo "No git remote set, aborting..." >&2
+  echo "Not on a git branch, aborting..." >&2
   exit 1
 fi
 

lib/kamal/cli/templates/sample_hooks/pre-traefik-reboot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooting Traefik on $KAMAL_HOSTS..."

lib/kamal/cli/traefik.rb

@@ -11,14 +11,23 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
 
   desc "reboot", "Reboot Traefik on servers (stop container, remove container, start new container)"
   option :rolling, type: :boolean, default: false, desc: "Reboot traefik on hosts in sequence, rather than in parallel"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def reboot
-    mutating do
+    confirming "This will cause a brief outage on each host. Are you sure?" do
-      on(KAMAL.traefik_hosts, in: options[:rolling] ? :sequence : :parallel) do
+      mutating do
-        execute *KAMAL.auditor.record("Rebooted traefik"), verbosity: :debug
+        host_groups = options[:rolling] ? KAMAL.traefik_hosts : [ KAMAL.traefik_hosts ]
-        execute *KAMAL.registry.login
+        host_groups.each do |hosts|
-        execute *KAMAL.traefik.stop
+          host_list = Array(hosts).join(",")
-        execute *KAMAL.traefik.remove_container
+          run_hook "pre-traefik-reboot", hosts: host_list
-        execute *KAMAL.traefik.run
+          on(hosts) do
+            execute *KAMAL.auditor.record("Rebooted traefik"), verbosity: :debug
+            execute *KAMAL.registry.login
+            execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
+            execute *KAMAL.traefik.remove_container
+            execute *KAMAL.traefik.run
+          end
+          run_hook "post-traefik-reboot", hosts: host_list
+        end
       end
     end
   end
@@ -38,7 +47,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
     mutating do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.auditor.record("Stopped traefik"), verbosity: :debug
-        execute *KAMAL.traefik.stop
+        execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
       end
     end
   end

lib/kamal/commander.rb

@@ -3,11 +3,13 @@ require "active_support/core_ext/module/delegation"
 
 class Kamal::Commander
   attr_accessor :verbosity, :holding_lock, :hold_lock_on_error
+  delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :traefik_hosts, :accessory_hosts, to: :specifics
 
   def initialize
     self.verbosity = :info
     self.holding_lock = false
     self.hold_lock_on_error = false
+    @specifics = nil
   end
 
   def config
@@ -24,57 +26,44 @@ class Kamal::Commander
   attr_reader :specific_roles, :specific_hosts
 
   def specific_primary!
-    self.specific_hosts = [ config.primary_web_host ]
+    @specifics = nil
+    self.specific_hosts = [ config.primary_host ]
   end
 
   def specific_roles=(role_names)
-    @specific_roles = config.roles.select { |r| role_names.include?(r.name) } if role_names.present?
+    @specifics = nil
+    if role_names.present?
+      @specific_roles = Kamal::Utils.filter_specific_items(role_names, config.roles)
+
+      if @specific_roles.empty?
+        raise ArgumentError, "No --roles match for #{role_names.join(',')}"
+      end
+
+      @specific_roles
+    end
   end
 
   def specific_hosts=(hosts)
-    @specific_hosts = config.all_hosts & hosts if hosts.present?
+    @specifics = nil
-  end
+    if hosts.present?
+      @specific_hosts = Kamal::Utils.filter_specific_items(hosts, config.all_hosts)
 
-  def primary_host
+      if @specific_hosts.empty?
-    specific_hosts&.first || specific_roles&.first&.primary_host || config.primary_web_host
+        raise ArgumentError, "No --hosts match for #{hosts.join(',')}"
-  end
+      end
 
-  def roles
+      @specific_hosts
-    (specific_roles || config.roles).select do |role|
-      ((specific_hosts || config.all_hosts) & role.hosts).any?
     end
   end
 
-  def hosts
-    (specific_hosts || config.all_hosts).select do |host|
-      (specific_roles || config.roles).flat_map(&:hosts).include?(host)
-    end
-  end
-
-  def boot_strategy
-    if config.boot.limit.present?
-      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
-    else
-      {}
-    end
-  end
-
-  def roles_on(host)
-    roles.select { |role| role.hosts.include?(host.to_s) }.map(&:name)
-  end
-
-  def traefik_hosts
-    specific_hosts || config.traefik_hosts
-  end
-
-  def accessory_hosts
-    specific_hosts || config.accessories.flat_map(&:hosts)
-  end
-
   def accessory_names
     config.accessories&.collect(&:name) || []
   end
 
+  def accessories_on(host)
+    config.accessories.select { |accessory| accessory.hosts.include?(host.to_s) }.map(&:name)
+  end
+
 
   def app(role: nil)
     Kamal::Commands::App.new(config, role: role)
@@ -116,10 +105,15 @@ class Kamal::Commander
     @registry ||= Kamal::Commands::Registry.new(config)
   end
 
+  def server
+    @server ||= Kamal::Commands::Server.new(config)
+  end
+
   def traefik
     @traefik ||= Kamal::Commands::Traefik.new(config)
   end
 
+
   def with_verbosity(level)
     old_level = self.verbosity
 
@@ -132,6 +126,14 @@ class Kamal::Commander
     SSHKit.config.output_verbosity = old_level
   end
 
+  def boot_strategy
+    if config.boot.limit.present?
+      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
+    else
+      {}
+    end
+  end
+
   def holding_lock?
     self.holding_lock
   end
@@ -151,4 +153,8 @@ class Kamal::Commander
       SSHKit.config.command_map[:docker] = "docker" # No need to use /usr/bin/env, just clogs up the logs
       SSHKit.config.output_verbosity = verbosity
     end
+
+    def specifics
+      @specifics ||= Kamal::Commander::Specifics.new(config, specific_hosts, specific_roles)
+    end
 end

lib/kamal/commander/specifics.rb

@@ -0,0 +1,49 @@
+class Kamal::Commander::Specifics
+  attr_reader :primary_host, :primary_role, :hosts, :roles
+  delegate :stable_sort!, to: Kamal::Utils
+
+  def initialize(config, specific_hosts, specific_roles)
+    @config, @specific_hosts, @specific_roles = config, specific_hosts, specific_roles
+
+    @roles, @hosts = specified_roles, specified_hosts
+
+    @primary_host = specific_hosts&.first || primary_specific_role&.primary_host || config.primary_host
+    @primary_role = primary_or_first_role(roles_on(primary_host))
+
+    stable_sort!(roles) { |role| role == primary_role ? 0 : 1 }
+    stable_sort!(hosts) { |host| roles_on(host).any? { |role| role == primary_role } ? 0 : 1 }
+  end
+
+  def roles_on(host)
+    roles.select { |role| role.hosts.include?(host.to_s) }
+  end
+
+  def traefik_hosts
+    specific_hosts || config.traefik_hosts
+  end
+
+  def accessory_hosts
+    specific_hosts || config.accessories.flat_map(&:hosts)
+  end
+
+  private
+    attr_reader :config, :specific_hosts, :specific_roles
+
+    def primary_specific_role
+      primary_or_first_role(specific_roles) if specific_roles.present?
+    end
+
+    def primary_or_first_role(roles)
+      roles.detect { |role| role == config.primary_role } || roles.first
+    end
+
+    def specified_roles
+      (specific_roles || config.roles) \
+        .select { |role| ((specific_hosts || config.all_hosts) & role.hosts).any? }
+    end
+
+    def specified_hosts
+      (specific_hosts || config.all_hosts) \
+        .select { |host| (specific_roles || config.roles).flat_map(&:hosts).include?(host) }
+    end
+end

lib/kamal/commands/accessory.rb

@@ -86,14 +86,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     end
   end
 
-  def make_directory_for(remote_file)
-    make_directory Pathname.new(remote_file).dirname.to_s
-  end
-
-  def make_directory(path)
-    [ :mkdir, "-p", path ]
-  end
-
   def remove_service_directory
     [ :rm, "-rf", service_name ]
   end
@@ -106,6 +98,14 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     docker :image, :rm, "--force", image
   end
 
+  def make_env_directory
+    make_directory accessory_config.env.secrets_directory
+  end
+
+  def remove_env_file
+    [ :rm, "-f", accessory_config.env.secrets_file ]
+  end
+
   private
     def service_filter
       [ "--filter", "label=service=#{service_name}" ]

lib/kamal/commands/app.rb

@@ -1,4 +1,6 @@
 class Kamal::Commands::App < Kamal::Commands::Base
+  include Assets, Containers, Cord, Execution, Images, Logging
+
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 
   attr_reader :role
@@ -8,23 +10,19 @@ class Kamal::Commands::App < Kamal::Commands::Base
     @role = role
   end
 
-  def start_or_run(hostname: nil)
-    combine start, run(hostname: hostname), by: "||"
-  end
-
   def run(hostname: nil)
-    role = config.role(self.role)
-
     docker :run,
       "--detach",
       "--restart unless-stopped",
       "--name", container_name,
-      *(["--hostname", hostname] if hostname),
+      *([ "--hostname", hostname ] if hostname),
       "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
+      "-e", "KAMAL_VERSION=\"#{config.version}\"",
       *role.env_args,
       *role.health_check_args,
-      *config.logging_args,
+      *role.logging_args,
       *config.volume_args,
+      *role.asset_volume_args,
       *role.label_args,
       *role.option_args,
       config.absolute_image,
@@ -50,55 +48,8 @@ class Kamal::Commands::App < Kamal::Commands::Base
   end
 
 
-  def logs(since: nil, lines: nil, grep: nil)
-    pipe \
-      current_running_container_id,
-      "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
-      ("grep '#{grep}'" if grep)
-  end
-
-  def follow_logs(host:, grep: nil)
-    run_over_ssh \
-      pipe(
-        current_running_container_id,
-        "xargs docker logs --timestamps --tail 10 --follow 2>&1",
-        (%(grep "#{grep}") if grep)
-      ),
-      host: host
-  end
-
-
-  def execute_in_existing_container(*command, interactive: false)
-    docker :exec,
-      ("-it" if interactive),
-      container_name,
-      *command
-  end
-
-  def execute_in_new_container(*command, interactive: false)
-    role = config.role(self.role)
-
-    docker :run,
-      ("-it" if interactive),
-      "--rm",
-      *config.env_args,
-      *config.volume_args,
-      *role&.option_args,
-      config.absolute_image,
-      *command
-  end
-
-  def execute_in_existing_container_over_ssh(*command, host:)
-    run_over_ssh execute_in_existing_container(*command, interactive: true), host: host
-  end
-
-  def execute_in_new_container_over_ssh(*command, host:)
-    run_over_ssh execute_in_new_container(*command, interactive: true), host: host
-  end
-
-
   def current_running_container_id
-    docker :ps, "--quiet", *filter_args(statuses: ACTIVE_DOCKER_STATUSES), "--latest"
+    current_running_container(format: "--quiet")
   end
 
   def container_id_for_version(version, only_running: false)
@@ -106,61 +57,57 @@ class Kamal::Commands::App < Kamal::Commands::Base
   end
 
   def current_running_version
-    list_versions("--latest", statuses: ACTIVE_DOCKER_STATUSES)
+    pipe \
+      current_running_container(format: "--format '{{.Names}}'"),
+      extract_version_from_name
   end
 
   def list_versions(*docker_args, statuses: nil)
     pipe \
       docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
-      %(while read line; do echo ${line##{service_role_dest}-}; done) # Extract SHA from "service-role-dest-SHA"
+      extract_version_from_name
   end
 
-  def list_containers
+
-    docker :container, :ls, "--all", *filter_args
+  def make_env_directory
+    make_directory role.env.secrets_directory
   end
 
-  def list_container_names
+  def remove_env_file
-    [ *list_containers, "--format", "'{{ .Names }}'" ]
+    [ :rm, "-f", role.env.secrets_file ]
-  end
-
-  def remove_container(version:)
-    pipe \
-      container_id_for(container_name: container_name(version)),
-      xargs(docker(:container, :rm))
-  end
-
-  def rename_container(version:, new_version:)
-    docker :rename, container_name(version), container_name(new_version)
-  end
-
-  def remove_containers
-    docker :container, :prune, "--force", *filter_args
-  end
-
-  def list_images
-    docker :image, :ls, config.repository
-  end
-
-  def remove_images
-    docker :image, :prune, "--all", "--force", *filter_args
-  end
-
-  def tag_current_as_latest
-    docker :tag, config.absolute_image, config.latest_image
   end
 
 
   private
     def container_name(version = nil)
-      [ config.service, role, config.destination, version || config.version ].compact.join("-")
+      [ role.container_prefix, version || config.version ].compact.join("-")
+    end
+
+    def latest_image_id
+      docker :image, :ls, *argumentize("--filter", "reference=#{config.latest_image}"), "--format", "'{{.ID}}'"
+    end
+
+    def current_running_container(format:)
+      pipe \
+        shell(chain(latest_image_container(format: format), latest_container(format: format))),
+        [ :head, "-1" ]
+    end
+
+    def latest_image_container(format:)
+      latest_container format: format, filters: [ "ancestor=$(#{latest_image_id.join(" ")})" ]
+    end
+
+    def latest_container(format:, filters: nil)
+      docker :ps, "--latest", *format, *filter_args(statuses: ACTIVE_DOCKER_STATUSES), argumentize("--filter", filters)
     end
 
     def filter_args(statuses: nil)
       argumentize "--filter", filters(statuses: statuses)
     end
 
-    def service_role_dest
+    def extract_version_from_name
-      [config.service, role, config.destination].compact.join("-")
+      # Extract SHA from "service-role-dest-SHA"
+      %(while read line; do echo ${line##{role.container_prefix}-}; done)
     end
 
     def filters(statuses: nil)

lib/kamal/commands/app/assets.rb

@@ -0,0 +1,51 @@
+module Kamal::Commands::App::Assets
+  def extract_assets
+    asset_container = "#{role.container_prefix}-assets"
+
+    combine \
+      make_directory(role.asset_extracted_path),
+      [ *docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true" ],
+      docker(:run, "--name", asset_container, "--detach", "--rm", config.absolute_image, "sleep 1000000"),
+      docker(:cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_path),
+      docker(:stop, "-t 1", asset_container),
+      by: "&&"
+  end
+
+  def sync_asset_volumes(old_version: nil)
+    new_extracted_path, new_volume_path = role.asset_extracted_path(config.version), role.asset_volume.host_path
+    if old_version.present?
+      old_extracted_path, old_volume_path = role.asset_extracted_path(old_version), role.asset_volume(old_version).host_path
+    end
+
+    commands = [ make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path) ]
+
+    if old_version.present?
+      commands << copy_contents(new_extracted_path, old_volume_path, continue_on_error: true)
+      commands << copy_contents(old_extracted_path, new_volume_path, continue_on_error: true)
+    end
+
+    chain *commands
+  end
+
+  def clean_up_assets
+    chain \
+      find_and_remove_older_siblings(role.asset_extracted_path),
+      find_and_remove_older_siblings(role.asset_volume_path)
+  end
+
+  private
+    def find_and_remove_older_siblings(path)
+      [
+        :find,
+        Pathname.new(path).dirname.to_s,
+        "-maxdepth 1",
+        "-name", "'#{role.container_prefix}-*'",
+        "!", "-name", Pathname.new(path).basename.to_s,
+        "-exec rm -rf \"{}\" +"
+      ]
+    end
+
+    def copy_contents(source, destination, continue_on_error: false)
+      [ :cp, "-rnT", "#{source}", destination, *("|| true" if continue_on_error) ]
+    end
+end

lib/kamal/commands/app/containers.rb

@@ -0,0 +1,23 @@
+module Kamal::Commands::App::Containers
+  def list_containers
+    docker :container, :ls, "--all", *filter_args
+  end
+
+  def list_container_names
+    [ *list_containers, "--format", "'{{ .Names }}'" ]
+  end
+
+  def remove_container(version:)
+    pipe \
+      container_id_for(container_name: container_name(version)),
+      xargs(docker(:container, :rm))
+  end
+
+  def rename_container(version:, new_version:)
+    docker :rename, container_name(version), container_name(new_version)
+  end
+
+  def remove_containers
+    docker :container, :prune, "--force", *filter_args
+  end
+end

lib/kamal/commands/app/cord.rb

@@ -0,0 +1,22 @@
+module Kamal::Commands::App::Cord
+  def cord(version:)
+    pipe \
+      docker(:inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", container_name(version)),
+      [ :awk, "'$2 == \"#{role.cord_volume.container_path}\" {print $1}'" ]
+  end
+
+  def tie_cord(cord)
+    create_empty_file(cord)
+  end
+
+  def cut_cord(cord)
+    remove_directory(cord)
+  end
+
+  private
+    def create_empty_file(file)
+      chain \
+        make_directory_for(file),
+        [ :touch, file ]
+    end
+end

lib/kamal/commands/app/execution.rb

@@ -0,0 +1,29 @@
+module Kamal::Commands::App::Execution
+  def execute_in_existing_container(*command, interactive: false, env:)
+    docker :exec,
+      ("-it" if interactive),
+      *argumentize("--env", env),
+      container_name,
+      *command
+  end
+
+  def execute_in_new_container(*command, interactive: false, env:)
+    docker :run,
+      ("-it" if interactive),
+      "--rm",
+      *role&.env_args,
+      *argumentize("--env", env),
+      *config.volume_args,
+      *role&.option_args,
+      config.absolute_image,
+      *command
+  end
+
+  def execute_in_existing_container_over_ssh(*command, host:, env:)
+    run_over_ssh execute_in_existing_container(*command, interactive: true, env: env), host: host
+  end
+
+  def execute_in_new_container_over_ssh(*command, host:, env:)
+    run_over_ssh execute_in_new_container(*command, interactive: true, env: env), host: host
+  end
+end

lib/kamal/commands/app/images.rb

@@ -0,0 +1,13 @@
+module Kamal::Commands::App::Images
+  def list_images
+    docker :image, :ls, config.repository
+  end
+
+  def remove_images
+    docker :image, :prune, "--all", "--force", *filter_args
+  end
+
+  def tag_latest_image
+    docker :tag, config.absolute_image, config.latest_image
+  end
+end

lib/kamal/commands/app/logging.rb

@@ -0,0 +1,18 @@
+module Kamal::Commands::App::Logging
+  def logs(since: nil, lines: nil, grep: nil)
+    pipe \
+      current_running_container_id,
+      "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
+      ("grep '#{grep}'" if grep)
+  end
+
+  def follow_logs(host:, lines: nil, grep: nil)
+    run_over_ssh \
+      pipe(
+        current_running_container_id,
+        "xargs docker logs --timestamps#{" --tail #{lines}" if lines} --follow 2>&1",
+        (%(grep "#{grep}") if grep)
+      ),
+      host: host
+  end
+end

lib/kamal/commands/auditor.rb

@@ -19,7 +19,9 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
 
   private
     def audit_log_file
-      [ "kamal", config.service, config.destination, "audit.log" ].compact.join("-")
+      file = [ config.service, config.destination, "audit.log" ].compact.join("-")
+
+      File.join(config.run_directory, file)
     end
 
     def audit_tags(**details)

lib/kamal/commands/base.rb

@@ -18,7 +18,7 @@ module Kamal::Commands
         elsif config.ssh.proxy && config.ssh.proxy.is_a?(Net::SSH::Proxy::Command)
           cmd << " -o ProxyCommand='#{config.ssh.proxy.command_line_template}'"
         end
-        cmd << " -t #{config.ssh.user}@#{host} '#{command.join(" ")}'"
+        cmd << " -t #{config.ssh.user}@#{host} -p #{config.ssh.port} '#{command.join(" ")}'"
       end
     end
 
@@ -26,6 +26,18 @@ module Kamal::Commands
       docker :container, :ls, *("--all" unless only_running), "--filter", "name=^#{container_name}$", "--quiet"
     end
 
+    def make_directory_for(remote_file)
+      make_directory Pathname.new(remote_file).dirname.to_s
+    end
+
+    def make_directory(path)
+      [ :mkdir, "-p", path ]
+    end
+
+    def remove_directory(path)
+      [ :rm, "-r", path ]
+    end
+
     private
       def combine(*commands, by: "&&")
         commands
@@ -50,14 +62,26 @@ module Kamal::Commands
         combine *commands, by: ">"
       end
 
+      def any(*commands)
+        combine *commands, by: "||"
+      end
+
       def xargs(command)
         [ :xargs, command ].flatten
       end
 
+      def shell(command)
+        [ :sh, "-c", "'#{command.flatten.join(" ").gsub("'", "'\\\\''")}'" ]
+      end
+
       def docker(*args)
         args.compact.unshift :docker
       end
 
+      def git(*args)
+        args.compact.unshift :git
+      end
+
       def tags(**details)
         Kamal::Tags.from_config(config, **details)
       end

lib/kamal/commands/builder.rb

@@ -1,7 +1,7 @@
 require "active_support/core_ext/string/filters"
 
 class Kamal::Commands::Builder < Kamal::Commands::Base
-  delegate :create, :remove, :push, :clean, :pull, :info, to: :target
+  delegate :create, :remove, :push, :clean, :pull, :info, :validate_image, to: :target
 
   def name
     target.class.to_s.remove("Kamal::Commands::Builder::").underscore.inquiry

lib/kamal/commands/builder/base.rb

@@ -3,7 +3,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
   class BuilderError < StandardError; end
 
   delegate :argumentize, to: Kamal::Utils
-  delegate :args, :secrets, :dockerfile, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, to: :builder_config
+  delegate :args, :secrets, :dockerfile, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, :ssh, :git_archive?, to: :builder_config
 
   def clean
     docker :image, :rm, "--force", config.absolute_image
@@ -13,14 +13,33 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
     docker :pull, config.absolute_image
   end
 
+  def push
+    if git_archive?
+      pipe \
+        git(:archive, "--format=tar", :HEAD),
+        build_and_push
+    else
+      build_and_push
+    end
+  end
+
   def build_options
-    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile ]
+    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_ssh ]
   end
 
   def build_context
     config.builder.context
   end
 
+  def validate_image
+    pipe \
+      docker(:inspect, "-f", "'{{ .Config.Labels.service }}'", config.absolute_image),
+      any(
+        [ :grep, "-x", config.service ],
+        "(echo \"Image #{config.absolute_image} is missing the 'service' label\" && exit 1)"
+      )
+  end
+
 
   private
     def build_tags
@@ -29,8 +48,8 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
 
     def build_cache
       if cache_to && cache_from
-        ["--cache-to", cache_to,
+        [ "--cache-to", cache_to,
-          "--cache-from", cache_from]
+          "--cache-from", cache_from ]
       end
     end
 
@@ -54,6 +73,10 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
       end
     end
 
+    def build_ssh
+      argumentize "--ssh", ssh if ssh.present?
+    end
+
     def builder_config
       config.builder
     end

lib/kamal/commands/builder/multiarch.rb

@@ -7,15 +7,6 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
     docker :buildx, :rm, builder_name
   end
 
-  def push
-    docker :buildx, :build,
-      "--push",
-      "--platform", "linux/amd64,linux/arm64",
-      "--builder", builder_name,
-      *build_options,
-      build_context
-  end
-
   def info
     combine \
       docker(:context, :ls),
@@ -26,4 +17,21 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
     def builder_name
       "kamal-#{config.service}-multiarch"
     end
+
+    def platform_names
+      if local_arch
+        "linux/#{local_arch}"
+      else
+        "linux/amd64,linux/arm64"
+      end
+    end
+
+    def build_and_push
+      docker :buildx, :build,
+        "--push",
+        "--platform", platform_names,
+        "--builder", builder_name,
+        *build_options,
+        build_context
+    end
 end

lib/kamal/commands/builder/native.rb

@@ -7,14 +7,15 @@ class Kamal::Commands::Builder::Native < Kamal::Commands::Builder::Base
     # No-op on native without cache
   end
 
-  def push
-    combine \
-      docker(:build, *build_options, build_context),
-      docker(:push, config.absolute_image),
-      docker(:push, config.latest_image)
-  end
-
   def info
     # No-op on native
   end
+
+  private
+    def build_and_push
+      combine \
+        docker(:build, *build_options, build_context),
+        docker(:push, config.absolute_image),
+        docker(:push, config.latest_image)
+    end
 end

lib/kamal/commands/builder/native/cached.rb

@@ -7,10 +7,11 @@ class Kamal::Commands::Builder::Native::Cached < Kamal::Commands::Builder::Nativ
     docker :buildx, :rm, builder_name
   end
 
-  def push
+  private
-    docker :buildx, :build,
+    def build_and_push
-      "--push",
+      docker :buildx, :build,
-      *build_options,
+        "--push",
-      build_context
+        *build_options,
-  end
+        build_context
+    end
 end

lib/kamal/commands/builder/native/remote.rb

@@ -11,15 +11,6 @@ class Kamal::Commands::Builder::Native::Remote < Kamal::Commands::Builder::Nativ
       remove_buildx
   end
 
-  def push
-    docker :buildx, :build,
-      "--push",
-      "--platform", platform,
-      "--builder", builder_name,
-      *build_options,
-      build_context
-  end
-
   def info
     chain \
       docker(:context, :ls),
@@ -56,4 +47,13 @@ class Kamal::Commands::Builder::Native::Remote < Kamal::Commands::Builder::Nativ
     def remove_buildx
       docker :buildx, :rm, builder_name
     end
+
+    def build_and_push
+      docker :buildx, :build,
+      "--push",
+      "--platform", platform,
+      "--builder", builder_name,
+      *build_options,
+      build_context
+    end
 end

lib/kamal/commands/docker.rb

@@ -1,7 +1,7 @@
 class Kamal::Commands::Docker < Kamal::Commands::Base
   # Install Docker using the https://github.com/docker/docker-install convenience script.
   def install
-    pipe [ :curl, "-fsSL", "https://get.docker.com" ], :sh
+    pipe get_docker, :sh
   end
 
   # Checks the Docker client version. Fails if Docker is not installed.
@@ -16,6 +16,15 @@ class Kamal::Commands::Docker < Kamal::Commands::Base
 
   # Do we have superuser access to install Docker and start system services?
   def superuser?
-    [ '[ "${EUID:-$(id -u)}" -eq 0 ]' ]
+    [ '[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null' ]
   end
+
+  private
+    def get_docker
+      shell \
+        any \
+          [ :curl, "-fsSL", "https://get.docker.com" ],
+          [ :wget, "-O -", "https://get.docker.com" ],
+          [ :echo, "\"exit 1\"" ]
+    end
 end

lib/kamal/commands/healthcheck.rb

@@ -1,21 +1,19 @@
 class Kamal::Commands::Healthcheck < Kamal::Commands::Base
-  EXPOSED_PORT = 3999
-
   def run
-    web = config.role(:web)
+    primary = config.role(config.primary_role)
 
     docker :run,
       "--detach",
       "--name", container_name_with_version,
-      "--publish", "#{EXPOSED_PORT}:#{config.healthcheck["port"]}",
+      "--publish", "#{exposed_port}:#{config.healthcheck["port"]}",
-      "--label", "service=#{container_name}",
+      "--label", "service=#{config.healthcheck_service}",
-      "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
+      "-e", "KAMAL_CONTAINER_NAME=\"#{config.healthcheck_service}\"",
-      *web.env_args,
+      *primary.env_args,
-      *web.health_check_args,
+      *primary.health_check_args(cord: false),
       *config.volume_args,
-      *web.option_args,
+      *primary.option_args,
       config.absolute_image,
-      web.cmd
+      primary.cmd
   end
 
   def status
@@ -27,7 +25,7 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
   end
 
   def logs
-    pipe container_id, xargs(docker(:logs, "--tail", 50, "2>&1"))
+    pipe container_id, xargs(docker(:logs, "--tail", log_lines, "2>&1"))
   end
 
   def stop
@@ -39,12 +37,8 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
   end
 
   private
-    def container_name
-      [ "healthcheck", config.service, config.destination ].compact.join("-")
-    end
-
     def container_name_with_version
-      "#{container_name}-#{config.version}"
+      "#{config.healthcheck_service}-#{config.version}"
     end
 
     def container_id
@@ -52,6 +46,14 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
     end
 
     def health_url
-      "http://localhost:#{EXPOSED_PORT}#{config.healthcheck["path"]}"
+      "http://localhost:#{exposed_port}#{config.healthcheck["path"]}"
+    end
+
+    def exposed_port
+      config.healthcheck["exposed_port"]
+    end
+
+    def log_lines
+      config.healthcheck["log_lines"]
     end
 end

lib/kamal/commands/hook.rb

@@ -9,6 +9,6 @@ class Kamal::Commands::Hook < Kamal::Commands::Base
 
   private
     def hook_file(hook)
-      "#{config.hooks_path}/#{hook}"
+      File.join(config.hooks_path, hook)
     end
 end

lib/kamal/commands/lock.rb

@@ -1,17 +1,18 @@
 require "active_support/duration"
 require "time"
+require "base64"
 
 class Kamal::Commands::Lock < Kamal::Commands::Base
   def acquire(message, version)
     combine \
-      [:mkdir, lock_dir],
+      [ :mkdir, lock_dir ],
       write_lock_details(message, version)
   end
 
   def release
     combine \
-      [:rm, lock_details_file],
+      [ :rm, lock_details_file ],
-      [:rm, "-r", lock_dir]
+      [ :rm, "-r", lock_dir ]
   end
 
   def status
@@ -20,31 +21,41 @@ class Kamal::Commands::Lock < Kamal::Commands::Base
       read_lock_details
   end
 
+  def ensure_locks_directory
+    [ :mkdir, "-p", locks_dir ]
+  end
+
   private
     def write_lock_details(message, version)
       write \
-        [:echo, "\"#{Base64.encode64(lock_details(message, version))}\""],
+        [ :echo, "\"#{Base64.encode64(lock_details(message, version))}\"" ],
         lock_details_file
     end
 
     def read_lock_details
       pipe \
-        [:cat, lock_details_file],
+        [ :cat, lock_details_file ],
-        [:base64, "-d"]
+        [ :base64, "-d" ]
     end
 
     def stat_lock_dir
       write \
-        [:stat, lock_dir],
+        [ :stat, lock_dir ],
         "/dev/null"
     end
 
+    def locks_dir
+      File.join(config.run_directory, "locks")
+    end
+
     def lock_dir
-      "kamal_lock-#{config.service}"
+      dir_name = [ config.service, config.destination ].compact.join("-")
+
+      File.join(locks_dir, dir_name)
     end
 
     def lock_details_file
-      [lock_dir, :details].join("/")
+      File.join(lock_dir, "details")
     end
 
     def lock_details(message, version)
@@ -56,7 +67,7 @@ class Kamal::Commands::Lock < Kamal::Commands::Base
     end
 
     def locked_by
-      `git config user.name`.strip
+      Kamal::Git.user_name
     rescue Errno::ENOENT
       "Unknown"
     end

lib/kamal/commands/prune.rb

@@ -3,7 +3,7 @@ require "active_support/core_ext/numeric/time"
 
 class Kamal::Commands::Prune < Kamal::Commands::Base
   def dangling_images
-    docker :image, :prune, "--force", "--filter", "label=service=#{config.service}", "--filter", "dangling=true"
+    docker :image, :prune, "--force", "--filter", "label=service=#{config.service}"
   end
 
   def tagged_images
@@ -13,16 +13,20 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
       "while read image tag; do docker rmi $tag; done"
   end
 
-  def containers(keep_last: 5)
+  def app_containers(retain:)
     pipe \
       docker(:ps, "-q", "-a", *service_filter, *stopped_containers_filters),
-      "tail -n +#{keep_last + 1}",
+      "tail -n +#{retain + 1}",
       "while read container_id; do docker rm $container_id; done"
   end
 
+  def healthcheck_containers
+    docker :container, :prune, "--force", *healthcheck_service_filter
+  end
+
   private
     def stopped_containers_filters
-      [ "created", "exited", "dead" ].flat_map { |status| ["--filter", "status=#{status}"] }
+      [ "created", "exited", "dead" ].flat_map { |status| [ "--filter", "status=#{status}" ] }
     end
 
     def active_image_list
@@ -35,4 +39,8 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
     def service_filter
       [ "--filter", "label=service=#{config.service}" ]
     end
+
+    def healthcheck_service_filter
+      [ "--filter", "label=service=#{config.healthcheck_service}" ]
+    end
 end

lib/kamal/commands/registry.rb

@@ -2,7 +2,10 @@ class Kamal::Commands::Registry < Kamal::Commands::Base
   delegate :registry, to: :config
 
   def login
-    docker :login, registry["server"], "-u", sensitive(lookup("username")), "-p", sensitive(lookup("password"))
+    docker :login,
+      registry["server"],
+      "-u", sensitive(Kamal::Utils.escape_shell_value(lookup("username"))),
+      "-p", sensitive(Kamal::Utils.escape_shell_value(lookup("password")))
   end
 
   def logout

lib/kamal/commands/server.rb

@@ -0,0 +1,5 @@
+class Kamal::Commands::Server < Kamal::Commands::Base
+  def ensure_run_directory
+    [ :mkdir, "-p", config.run_directory ]
+  end
+end

lib/kamal/commands/traefik.rb

@@ -1,10 +1,18 @@
 class Kamal::Commands::Traefik < Kamal::Commands::Base
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
-  DEFAULT_IMAGE = "traefik:v2.9"
+  DEFAULT_IMAGE = "traefik:v2.10"
   CONTAINER_PORT = 80
   DEFAULT_ARGS = {
-    'log.level' => 'DEBUG'
+    "log.level" => "DEBUG"
+  }
+  DEFAULT_LABELS = {
+    # These ensure we serve a 502 rather than a 404 if no containers are available
+    "traefik.http.routers.catchall.entryPoints" => "http",
+    "traefik.http.routers.catchall.rule" => "PathPrefix(`/`)",
+    "traefik.http.routers.catchall.service" => "unavailable",
+    "traefik.http.routers.catchall.priority" => 1,
+    "traefik.http.services.unavailable.loadbalancer.server.port" => "0"
   }
 
   def run
@@ -31,7 +39,7 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
   end
 
   def start_or_run
-    combine start, run, by: "||"
+    any start, run
   end
 
   def info
@@ -63,6 +71,20 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
     "#{host_port}:#{CONTAINER_PORT}"
   end
 
+  def env
+    Kamal::Configuration::Env.from_config \
+      config: config.traefik.fetch("env", {}),
+      secrets_file: File.join(config.host_env_directory, "traefik", "traefik.env")
+  end
+
+  def make_env_directory
+    make_directory(env.secrets_directory)
+  end
+
+  def remove_env_file
+    [ :rm, "-f", env.secrets_file ]
+  end
+
   private
     def publish_args
       argumentize "--publish", port unless config.traefik["publish"] == false
@@ -73,17 +95,11 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
     end
 
     def env_args
-      env_config = config.traefik["env"] || {}
+      env.args
-
-      if env_config.present?
-        argumentize_env_with_secrets(env_config)
-      else
-        []
-      end
     end
 
     def labels
-      config.traefik["labels"] || []
+      DEFAULT_LABELS.merge(config.traefik["labels"] || {})
     end
 
     def image

lib/kamal/configuration.rb

@@ -6,11 +6,10 @@ require "erb"
 require "net/ssh/proxy/jump"
 
 class Kamal::Configuration
-  delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
+  delegate :service, :image, :servers, :labels, :registry, :stop_wait_time, :hooks_path, :logging, to: :raw_config, allow_nil: true
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
-  attr_accessor :destination
+  attr_reader :destination, :raw_config
-  attr_accessor :raw_config
 
   class << self
     def create_from(config_file:, destination: nil, version: nil)
@@ -26,7 +25,9 @@ class Kamal::Configuration
 
       def load_config_file(file)
         if file.exist?
-          YAML.load(ERB.new(IO.read(file)).result).symbolize_keys
+          # Newer Psych doesn't load aliases by default
+          load_method = YAML.respond_to?(:unsafe_load) ? :unsafe_load : :load
+          YAML.send(load_method, ERB.new(IO.read(file)).result).symbolize_keys
         else
           raise "Configuration file not found in #{file}"
         end
@@ -54,12 +55,25 @@ class Kamal::Configuration
   end
 
   def abbreviated_version
-    Kamal::Utils.abbreviate_version(version)
+    if version
+      # Don't abbreviate <sha>_uncommitted_<etc>
+      if version.include?("_")
+        version
+      else
+        version[0...7]
+      end
+    end
+  end
+
+  def minimum_version
+    raw_config.minimum_version
   end
 
 
   def roles
-    @roles ||= role_names.collect { |role_name| Role.new(role_name, config: self) }
+    @roles ||= role_names.collect do |role_name|
+      Role.new(role_name, config: self, specializations: role_specializations(role_name), primary: role_name == primary_role_name)
+    end
   end
 
   def role(name)
@@ -76,22 +90,37 @@ class Kamal::Configuration
 
 
   def all_hosts
-    roles.flat_map(&:hosts).uniq
+    (roles + accessories).flat_map(&:hosts).uniq
   end
 
-  def primary_web_host
+  def primary_host
-    role(:web).primary_host
+    primary_role&.primary_host
+  end
+
+  def primary_role_name
+    raw_config.primary_role || "web"
+  end
+
+  def primary_role
+    role(primary_role_name)
+  end
+
+  def allow_empty_roles?
+    raw_config.allow_empty_roles
+  end
+
+  def traefik_roles
+    roles.select(&:running_traefik?)
+  end
+
+  def traefik_role_names
+    traefik_roles.flat_map(&:name)
   end
 
   def traefik_hosts
-    roles.select(&:running_traefik?).flat_map(&:hosts).uniq
+    traefik_roles.flat_map(&:hosts).uniq
   end
 
-  def boot
-    Kamal::Configuration::Boot.new(config: self)
-  end
-
-
   def repository
     [ raw_config.registry["server"], image ].compact.join("/")
   end
@@ -101,22 +130,26 @@ class Kamal::Configuration
   end
 
   def latest_image
-    "#{repository}:latest"
+    "#{repository}:#{latest_tag}"
+  end
+
+  def latest_tag
+    [ "latest", *destination ].join("-")
   end
 
   def service_with_version
     "#{service}-#{version}"
   end
 
-
+  def require_destination?
-  def env_args
+    raw_config.require_destination
-    if raw_config.env.present?
-      argumentize_env_with_secrets(raw_config.env)
-    else
-      []
-    end
   end
 
+  def retain_containers
+    raw_config.retain_containers || 5
+  end
+
+
   def volume_args
     if raw_config.volumes.present?
       argumentize "--volume", raw_config.volumes
@@ -126,15 +159,27 @@ class Kamal::Configuration
   end
 
   def logging_args
-    if raw_config.logging.present?
+    if logging.present?
-      optionize({ "log-driver" => raw_config.logging["driver"] }.compact) +
+      optionize({ "log-driver" => logging["driver"] }.compact) +
-        argumentize("--log-opt", raw_config.logging["options"])
+        argumentize("--log-opt", logging["options"])
     else
       argumentize("--log-opt", { "max-size" => "10m" })
     end
   end
 
 
+  def boot
+    Kamal::Configuration::Boot.new(config: self)
+  end
+
+  def builder
+    Kamal::Configuration::Builder.new(config: self)
+  end
+
+  def traefik
+    raw_config.traefik || {}
+  end
+
   def ssh
     Kamal::Configuration::Ssh.new(config: self)
   end
@@ -145,32 +190,65 @@ class Kamal::Configuration
 
 
   def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7 }.merge(raw_config.healthcheck || {})
+    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }.merge(raw_config.healthcheck || {})
+  end
+
+  def healthcheck_service
+    [ "healthcheck", service, destination ].compact.join("-")
   end
 
   def readiness_delay
     raw_config.readiness_delay || 7
   end
 
-  def minimum_version
+  def run_id
-    raw_config.minimum_version
+    @run_id ||= SecureRandom.hex(16)
   end
 
+
+  def run_directory
+    raw_config.run_directory || ".kamal"
+  end
+
+  def run_directory_as_docker_volume
+    if Pathname.new(run_directory).absolute?
+      run_directory
+    else
+      File.join "$(pwd)", run_directory
+    end
+  end
+
+  def hooks_path
+    raw_config.hooks_path || ".kamal/hooks"
+  end
+
+  def asset_path
+    raw_config.asset_path
+  end
+
+
+  def host_env_directory
+    File.join(run_directory, "env")
+  end
+
+  def env
+    raw_config.env || {}
+  end
+
+
   def valid?
-    ensure_required_keys_present && ensure_valid_kamal_version
+    ensure_destination_if_required && ensure_required_keys_present && ensure_valid_kamal_version && ensure_retain_containers_valid && ensure_valid_service_name
   end
 
-
   def to_h
     {
       roles: role_names,
       hosts: all_hosts,
-      primary_host: primary_web_host,
+      primary_host: primary_host,
       version: version,
       repository: repository,
       absolute_image: absolute_image,
       service_with_version: service_with_version,
-      env_args: env_args,
       volume_args: volume_args,
       ssh_options: ssh.to_h,
       sshkit: sshkit.to_h,
@@ -181,28 +259,17 @@ class Kamal::Configuration
     }.compact
   end
 
-  def traefik
-    raw_config.traefik || {}
-  end
-
-  def hooks_path
-    raw_config.hooks_path || ".kamal/hooks"
-  end
-
-  def builder
-    Kamal::Configuration::Builder.new(config: self)
-  end
-
-  # Will raise KeyError if any secret ENVs are missing
-  def ensure_env_available
-    env_args
-    roles.each(&:env_args)
-
-    true
-  end
 
   private
     # Will raise ArgumentError if any required config keys are missing
+    def ensure_destination_if_required
+      if require_destination? && destination.nil?
+        raise ArgumentError, "You must specify a destination"
+      end
+
+      true
+    end
+
     def ensure_required_keys_present
       %i[ service image registry servers ].each do |key|
         raise ArgumentError, "Missing required configuration for #{key}" unless raw_config[key].present?
@@ -216,15 +283,31 @@ class Kamal::Configuration
         raise ArgumentError, "You must specify a password for the registry in config/deploy.yml (or set the ENV variable if that's used)"
       end
 
-      roles.each do |role|
+      unless role_names.include?(primary_role_name)
-        if role.hosts.empty?
+        raise ArgumentError, "The primary_role #{primary_role_name} isn't defined"
-          raise ArgumentError, "No servers specified for the #{role.name} role"
+      end
+
+      if primary_role.hosts.empty?
+        raise ArgumentError, "No servers specified for the #{primary_role.name} primary_role"
+      end
+
+      unless allow_empty_roles?
+        roles.each do |role|
+          if role.hosts.empty?
+            raise ArgumentError, "No servers specified for the #{role.name} role. You can ignore this with allow_empty_roles: true"
+          end
         end
       end
 
       true
     end
 
+    def ensure_valid_service_name
+      raise ArgumentError, "Service name can only include alphanumeric characters, hyphens, and underscores" unless raw_config[:service] =~ /^[a-z0-9_-]+$/i
+
+      true
+    end
+
     def ensure_valid_kamal_version
       if minimum_version && Gem::Version.new(minimum_version) > Gem::Version.new(Kamal::VERSION)
         raise ArgumentError, "Current version is #{Kamal::VERSION}, minimum required is #{minimum_version}"
@@ -233,17 +316,32 @@ class Kamal::Configuration
       true
     end
 
+    def ensure_retain_containers_valid
+      raise ArgumentError, "Must retain at least 1 container" if retain_containers < 1
+
+      true
+    end
+
 
     def role_names
       raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort
     end
 
+    def role_specializations(name)
+      if servers.is_a?(Array) || servers[name].is_a?(Array)
+        {}
+      else
+        servers[name].except("hosts")
+      end
+    end
+
     def git_version
       @git_version ||=
-        if system("git rev-parse")
+        if Kamal::Git.used?
-          uncommitted_suffix = Kamal::Utils.uncommitted_changes.present? ? "_uncommitted_#{SecureRandom.hex(8)}" : ""
+          if Kamal::Git.uncommitted_changes.present? && !builder.git_archive?
-
+            uncommitted_suffix = "_uncommitted_#{SecureRandom.hex(8)}"
-          "#{`git rev-parse HEAD`.strip}#{uncommitted_suffix}"
+          end
+          [ Kamal::Git.revision, uncommitted_suffix ].compact.join
         else
           raise "Can't use commit hash as version, no git repository found in #{Dir.pwd}"
         end

lib/kamal/configuration/accessory.rb

@@ -1,5 +1,5 @@
 class Kamal::Configuration::Accessory
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_accessor :name, :specifics
 
@@ -8,7 +8,7 @@ class Kamal::Configuration::Accessory
   end
 
   def service_name
-    "#{config.service}-#{name}"
+    specifics["service"] || "#{config.service}-#{name}"
   end
 
   def image
@@ -16,7 +16,7 @@ class Kamal::Configuration::Accessory
   end
 
   def hosts
-    if (specifics.keys & ["host", "hosts", "roles"]).size != 1
+    if (specifics.keys & [ "host", "hosts", "roles" ]).size != 1
       raise ArgumentError, "Specify one of `host`, `hosts` or `roles` for accessory `#{name}`"
     end
 
@@ -42,11 +42,13 @@ class Kamal::Configuration::Accessory
   end
 
   def env
-    specifics["env"] || {}
+    Kamal::Configuration::Env.from_config \
+      config: specifics.fetch("env", {}),
+      secrets_file: File.join(config.host_env_directory, "accessories", "#{service_name}.env")
   end
 
   def env_args
-    argumentize_env_with_secrets env
+    env.args
   end
 
   def files
@@ -58,8 +60,8 @@ class Kamal::Configuration::Accessory
 
   def directories
     specifics["directories"]&.to_h do |host_to_container_mapping|
-      host_relative_path, container_path = host_to_container_mapping.split(":")
+      host_path, container_path = host_to_container_mapping.split(":")
-      [ expand_host_path(host_relative_path), container_path ]
+      [ expand_host_path(host_path), container_path ]
     end || {}
   end
 
@@ -99,10 +101,10 @@ class Kamal::Configuration::Accessory
     end
 
     def with_clear_env_loaded
-      (env["clear"] || env).each { |k, v| ENV[k] = v }
+      env.clear.each { |k, v| ENV[k] = v }
       yield
     ensure
-      (env["clear"] || env).each { |k, v| ENV.delete(k) }
+      env.clear.each { |k, v| ENV.delete(k) }
     end
 
     def read_dynamic_file(local_file)
@@ -126,13 +128,17 @@ class Kamal::Configuration::Accessory
 
     def remote_directories_as_volumes
       specifics["directories"]&.collect do |host_to_container_mapping|
-        host_relative_path, container_path = host_to_container_mapping.split(":")
+        host_path, container_path = host_to_container_mapping.split(":")
-        [ expand_host_path(host_relative_path), container_path ].join(":")
+        [ expand_host_path(host_path), container_path ].join(":")
       end || []
     end
 
-    def expand_host_path(host_relative_path)
+    def expand_host_path(host_path)
-      "#{service_data_directory}/#{host_relative_path}"
+      absolute_path?(host_path) ? host_path : File.join(service_data_directory, host_path)
+    end
+
+    def absolute_path?(path)
+      Pathname.new(path).absolute?
     end
 
     def service_data_directory
@@ -143,7 +149,7 @@ class Kamal::Configuration::Accessory
       if specifics.key?("host")
         host = specifics["host"]
         if host
-          [host]
+          [ host ]
         else
           raise ArgumentError, "Missing host for accessory `#{name}`"
         end

lib/kamal/configuration/boot.rb

@@ -8,7 +8,7 @@ class Kamal::Configuration::Boot
     limit = @options["limit"]
 
     if limit.to_s.end_with?("%")
-      @host_count * limit.to_i / 100
+      [ @host_count * limit.to_i / 100, 1 ].max
     else
       limit
     end

lib/kamal/configuration/builder.rb

@@ -40,7 +40,7 @@ class Kamal::Configuration::Builder
   end
 
   def context
-    @options["context"] || "."
+    @options["context"] || (git_archive? ? "-" : ".")
   end
 
   def local_arch
@@ -81,10 +81,18 @@ class Kamal::Configuration::Builder
     end
   end
 
+  def ssh
+    @options["ssh"]
+  end
+
+  def git_archive?
+    Kamal::Git.used? && @options["context"].nil?
+  end
+
   private
     def valid?
       if @options["cache"] && @options["cache"]["type"]
-        raise ArgumentError, "Invalid cache type: #{@options["cache"]["type"]}" unless ["gha", "registry"].include?(@options["cache"]["type"])
+        raise ArgumentError, "Invalid cache type: #{@options["cache"]["type"]}" unless [ "gha", "registry" ].include?(@options["cache"]["type"])
       end
     end
 
@@ -105,7 +113,7 @@ class Kamal::Configuration::Builder
     end
 
     def cache_to_config_for_gha
-      [ "type=gha", @options["cache"]&.fetch("options", nil)].compact.join(",")
+      [ "type=gha", @options["cache"]&.fetch("options", nil) ].compact.join(",")
     end
 
     def cache_to_config_for_registry

lib/kamal/configuration/env.rb

@@ -0,0 +1,40 @@
+class Kamal::Configuration::Env
+  attr_reader :secrets_keys, :clear, :secrets_file
+  delegate :argumentize, to: Kamal::Utils
+
+  def self.from_config(config:, secrets_file: nil)
+    secrets_keys = config.fetch("secret", [])
+    clear = config.fetch("clear", config.key?("secret") ? {} : config)
+
+    new clear: clear, secrets_keys: secrets_keys, secrets_file: secrets_file
+  end
+
+  def initialize(clear:, secrets_keys:, secrets_file:)
+    @clear = clear
+    @secrets_keys = secrets_keys
+    @secrets_file = secrets_file
+  end
+
+  def args
+    [ "--env-file", secrets_file, *argumentize("--env", clear) ]
+  end
+
+  def secrets_io
+    StringIO.new(Kamal::EnvFile.new(secrets).to_s)
+  end
+
+  def secrets
+    @secrets ||= secrets_keys.to_h { |key| [ key, ENV.fetch(key) ] }
+  end
+
+  def secrets_directory
+    File.dirname(secrets_file)
+  end
+
+  def merge(other)
+    self.class.new \
+      clear: @clear.merge(other.clear),
+      secrets_keys: @secrets_keys | other.secrets_keys,
+      secrets_file: secrets_file
+  end
+end

lib/kamal/configuration/role.rb

@@ -1,10 +1,12 @@
 class Kamal::Configuration::Role
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
+  CORD_FILE = "cord"
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
-  attr_accessor :name
+  attr_reader :name
+  alias to_s name
 
-  def initialize(name, config:)
+  def initialize(name, config:, specializations:, primary:)
-   @name, @config = name.inquiry, config
+    @name, @config, @specializations, @primary = name.inquiry, config, specializations, primary
   end
 
   def primary_host
@@ -15,48 +17,6 @@ class Kamal::Configuration::Role
     @hosts ||= extract_hosts_from_config
   end
 
-  def labels
-    default_labels.merge(traefik_labels).merge(custom_labels)
-  end
-
-  def label_args
-    argumentize "--label", labels
-  end
-
-  def env
-    if config.env && config.env["secret"]
-      merged_env_with_secrets
-    else
-      merged_env
-    end
-  end
-
-  def env_args
-    argumentize_env_with_secrets env
-  end
-
-  def health_check_args
-    if health_check_cmd.present?
-      optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval })
-    else
-      []
-    end
-  end
-
-  def health_check_cmd
-    options = specializations["healthcheck"] || {}
-    options = config.healthcheck.merge(options) if running_traefik?
-
-    options["cmd"] || http_health_check(port: options["port"], path: options["path"])
-  end
-
-  def health_check_interval
-    options = specializations["healthcheck"] || {}
-    options = config.healthcheck.merge(options) if running_traefik?
-
-    options["interval"] || "1s"
-  end
-
   def cmd
     specializations["cmd"]
   end
@@ -69,12 +29,148 @@ class Kamal::Configuration::Role
     end
   end
 
+  def labels
+    default_labels.merge(traefik_labels).merge(custom_labels)
+  end
+
+  def label_args
+    argumentize "--label", labels
+  end
+
+  def logging_args
+    args = config.logging || {}
+    args.deep_merge!(specializations["logging"]) if specializations["logging"].present?
+
+    if args.any?
+      optionize({ "log-driver" => args["driver"] }.compact) +
+        argumentize("--log-opt", args["options"])
+    else
+      config.logging_args
+    end
+  end
+
+
+  def env
+    @env ||= base_env.merge(specialized_env)
+  end
+
+  def env_args
+    env.args
+  end
+
+  def asset_volume_args
+    asset_volume&.docker_args
+  end
+
+
+  def health_check_args(cord: true)
+    if health_check_cmd.present?
+      if cord && uses_cord?
+        optionize({ "health-cmd" => health_check_cmd_with_cord, "health-interval" => health_check_interval })
+          .concat(cord_volume.docker_args)
+      else
+        optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval })
+      end
+    else
+      []
+    end
+  end
+
+  def health_check_cmd
+    health_check_options["cmd"] || http_health_check(port: health_check_options["port"], path: health_check_options["path"])
+  end
+
+  def health_check_cmd_with_cord
+    "(#{health_check_cmd}) && (stat #{cord_container_file} > /dev/null || exit 1)"
+  end
+
+  def health_check_interval
+    health_check_options["interval"] || "1s"
+  end
+
+
   def running_traefik?
-    name.web? || specializations["traefik"]
+    if specializations["traefik"].nil?
+      primary?
+    else
+      specializations["traefik"]
+    end
+  end
+
+  def primary?
+    @primary
+  end
+
+
+  def uses_cord?
+    running_traefik? && cord_volume && health_check_cmd.present?
+  end
+
+  def cord_host_directory
+    File.join config.run_directory_as_docker_volume, "cords", [ container_prefix, config.run_id ].join("-")
+  end
+
+  def cord_volume
+    if (cord = health_check_options["cord"])
+      @cord_volume ||= Kamal::Configuration::Volume.new \
+        host_path: File.join(config.run_directory, "cords", [ container_prefix, config.run_id ].join("-")),
+        container_path: cord
+    end
+  end
+
+  def cord_host_file
+    File.join cord_volume.host_path, CORD_FILE
+  end
+
+  def cord_container_directory
+    health_check_options.fetch("cord", nil)
+  end
+
+  def cord_container_file
+    File.join cord_volume.container_path, CORD_FILE
+  end
+
+
+  def container_name(version = nil)
+    [ container_prefix, version || config.version ].compact.join("-")
+  end
+
+  def container_prefix
+    [ config.service, name, config.destination ].compact.join("-")
+  end
+
+
+  def asset_path
+    specializations["asset_path"] || config.asset_path
+  end
+
+  def assets?
+    asset_path.present? && running_traefik?
+  end
+
+  def asset_volume(version = nil)
+    if assets?
+      Kamal::Configuration::Volume.new \
+        host_path: asset_volume_path(version), container_path: asset_path
+    end
+  end
+
+  def asset_extracted_path(version = nil)
+    File.join config.run_directory, "assets", "extracted", container_name(version)
+  end
+
+  def asset_volume_path(version = nil)
+    File.join config.run_directory, "assets", "volumes", container_name(version)
+  end
+
+  def previous_roles
+    previous_role_names.map do |role_name|
+      Kamal::Configuration::Role.new(role_name, config: config, specializations: specializations, primary: primary?)
+    end
   end
 
   private
-    attr_accessor :config
+    attr_reader :config, :specializations
 
     def extract_hosts_from_config
       if config.servers.is_a?(Array)
@@ -86,11 +182,7 @@ class Kamal::Configuration::Role
     end
 
     def default_labels
-      if config.destination
+      { "service" => config.service, "role" => name, "destination" => config.destination }
-        { "service" => config.service, "role" => name, "destination" => config.destination }
-      else
-        { "service" => config.service, "role" => name }
-      end
     end
 
     def traefik_labels
@@ -100,6 +192,7 @@ class Kamal::Configuration::Role
           "traefik.http.services.#{traefik_service}.loadbalancer.server.scheme" => "http",
 
           "traefik.http.routers.#{traefik_service}.rule" => "PathPrefix(`/`)",
+          "traefik.http.routers.#{traefik_service}.priority" => "2",
           "traefik.http.middlewares.#{traefik_service}-retry.retry.attempts" => "5",
           "traefik.http.middlewares.#{traefik_service}-retry.retry.initialinterval" => "500ms",
           "traefik.http.routers.#{traefik_service}.middlewares" => "#{traefik_service}-retry@docker"
@@ -110,7 +203,7 @@ class Kamal::Configuration::Role
     end
 
     def traefik_service
-      [ config.service, name, config.destination ].compact.join("-")
+      container_prefix
     end
 
     def custom_labels
@@ -120,36 +213,30 @@ class Kamal::Configuration::Role
       end
     end
 
-    def specializations
-      if config.servers.is_a?(Array) || config.servers[name].is_a?(Array)
-        { }
-      else
-        config.servers[name].except("hosts")
-      end
-    end
-
     def specialized_env
-      specializations["env"] || {}
+      Kamal::Configuration::Env.from_config config: specializations.fetch("env", {})
-    end
-
-    def merged_env
-      config.env&.merge(specialized_env) || {}
     end
 
     # Secrets are stored in an array, which won't merge by default, so have to do it by hand.
-    def merged_env_with_secrets
+    def base_env
-      merged_env.tap do |new_env|
+      Kamal::Configuration::Env.from_config \
-        new_env["secret"] = Array(config.env["secret"]) + Array(specialized_env["secret"])
+        config: config.env,
-
+        secrets_file: File.join(config.host_env_directory, "roles", "#{container_prefix}.env")
-        # If there's no secret/clear split, everything is clear
-        clear_app_env  = config.env["secret"] ? Array(config.env["clear"]) : Array(config.env["clear"] || config.env)
-        clear_role_env = specialized_env["secret"] ? Array(specialized_env["clear"]) : Array(specialized_env["clear"] || specialized_env)
-
-        new_env["clear"] = (clear_app_env + clear_role_env).uniq
-      end
     end
 
     def http_health_check(port:, path:)
       "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present?
     end
+
+    def health_check_options
+      @health_check_options ||= begin
+        options = specializations["healthcheck"] || {}
+        options = config.healthcheck.merge(options) if running_traefik?
+        options
+      end
+    end
+
+    def previous_role_names
+      specializations.fetch("previously", [])
+    end
 end

lib/kamal/configuration/ssh.rb

@@ -9,6 +9,10 @@ class Kamal::Configuration::Ssh
     config.fetch("user", "root")
   end
 
+  def port
+    config.fetch("port", 22)
+  end
+
   def proxy
     if (proxy = config["proxy"])
       Net::SSH::Proxy::Jump.new(proxy.include?("@") ? proxy : "root@#{proxy}")
@@ -18,7 +22,7 @@ class Kamal::Configuration::Ssh
   end
 
   def options
-    { user: user, proxy: proxy, auth_methods: [ "publickey" ], logger: logger, keepalive: true, keepalive_interval: 30 }.compact
+    { user: user, port: port, proxy: proxy, logger: logger, keepalive: true, keepalive_interval: 30 }.compact
   end
 
   def to_h

lib/kamal/configuration/volume.rb

@@ -0,0 +1,22 @@
+class Kamal::Configuration::Volume
+  attr_reader :host_path, :container_path
+  delegate :argumentize, to: Kamal::Utils
+
+  def initialize(host_path:, container_path:)
+    @host_path = host_path
+    @container_path = container_path
+  end
+
+  def docker_args
+    argumentize "--volume", "#{host_path_for_docker_volume}:#{container_path}"
+  end
+
+  private
+    def host_path_for_docker_volume
+      if Pathname.new(host_path).absolute?
+        host_path
+      else
+        File.join "$(pwd)", host_path
+      end
+    end
+end

lib/kamal/env_file.rb

@@ -0,0 +1,38 @@
+# Encode an env hash as a string where secret values have been looked up and all values escaped for Docker.
+class Kamal::EnvFile
+  def initialize(env)
+    @env = env
+  end
+
+  def to_s
+    env_file = StringIO.new.tap do |contents|
+      @env.each do |key, value|
+        contents << docker_env_file_line(key, value)
+      end
+    end.string
+
+    # Ensure the file has some contents to avoid the SSHKIT empty file warning
+    env_file.presence || "\n"
+  end
+
+  alias to_str to_s
+
+  private
+    def docker_env_file_line(key, value)
+      "#{key}=#{escape_docker_env_file_value(value)}\n"
+    end
+
+    # Escape a value to make it safe to dump in a docker file.
+    def escape_docker_env_file_value(value)
+      # keep non-ascii(UTF-8) characters as it is
+      value.to_s.scan(/[\x00-\x7F]+|[^\x00-\x7F]+/).map do |part|
+        part.ascii_only? ? escape_docker_env_file_ascii_value(part) : part
+      end.join
+    end
+
+    def escape_docker_env_file_ascii_value(value)
+      # Doublequotes are treated literally in docker env files
+      # so remove leading and trailing ones and unescape any others
+      value.to_s.dump[1..-2].gsub(/\\"/, "\"")
+    end
+end

lib/kamal/git.rb

@@ -0,0 +1,19 @@
+module Kamal::Git
+  extend self
+
+  def used?
+    system("git rev-parse")
+  end
+
+  def user_name
+    `git config user.name`.strip
+  end
+
+  def revision
+    `git rev-parse HEAD`.strip
+  end
+
+  def uncommitted_changes
+    `git status --porcelain`.strip
+  end
+end

lib/kamal/sshkit_with_ext.rb

@@ -1,5 +1,6 @@
 require "sshkit"
 require "sshkit/dsl"
+require "net/scp"
 require "active_support/core_ext/hash/deep_merge"
 require "json"
 

lib/kamal/utils.rb

@@ -9,23 +9,13 @@ module Kamal::Utils
       if value.present?
         attr = "#{key}=#{escape_shell_value(value)}"
         attr = self.sensitive(attr, redaction: "#{key}=[REDACTED]") if sensitive
-        [ argument, attr]
+        [ argument, attr ]
       else
         [ argument, key ]
       end
     end
   end
 
-  # Return a list of shell arguments using the same named argument against the passed attributes,
-  # but redacts and expands secrets.
-  def argumentize_env_with_secrets(env)
-    if (secrets = env["secret"]).present?
-      argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, sensitive: true) + argumentize("-e", env["clear"])
-    else
-      argumentize "-e", env.fetch("clear", env)
-    end
-  end
-
   # Returns a list of shell-dashed option arguments. If the value is true, it's treated like a value-less option.
   def optionize(args, with: nil)
     options = if with
@@ -39,7 +29,7 @@ module Kamal::Utils
 
   # Flattens a one-to-many structure into an array of two-element arrays each containing a key-value pair
   def flatten_args(args)
-    args.flat_map { |key, value| value.try(:map) { |entry| [key, entry] } || [ [ key, value ] ] }
+    args.flat_map { |key, value| value.try(:map) { |entry| [ key, entry ] } || [ [ key, value ] ] }
   end
 
   # Marks sensitive values for redaction in logs and human-visible output.
@@ -62,19 +52,6 @@ module Kamal::Utils
     end
   end
 
-  def unredacted(value)
-    case
-    when value.respond_to?(:unredacted)
-      value.unredacted
-    when value.respond_to?(:transform_values)
-      value.transform_values { |value| unredacted value }
-    when value.respond_to?(:map)
-      value.map { |element| unredacted element }
-    else
-      value
-    end
-  end
-
   # Escape a value to make it safe for shell use.
   def escape_shell_value(value)
     value.to_s.dump
@@ -82,19 +59,23 @@ module Kamal::Utils
       .gsub(DOLLAR_SIGN_WITHOUT_SHELL_EXPANSION_REGEX, '\$')
   end
 
-  # Abbreviate a git revhash for concise display
+  # Apply a list of host or role filters, including wildcard matches
-  def abbreviate_version(version)
+  def filter_specific_items(filters, items)
-    if version
+    matches = []
-      # Don't abbreviate <sha>_uncommitted_<etc>
+
-      if version.include?("_")
+    Array(filters).select do |filter|
-        version
+      matches += Array(items).select do |item|
-      else
+        # Only allow * for a wildcard
-        version[0...7]
+        pattern = Regexp.escape(filter).gsub('\*', ".*")
+        # items are roles or hosts
+        (item.respond_to?(:name) ? item.name : item).match(/^#{pattern}$/)
       end
     end
+
+    matches
   end
 
-  def uncommitted_changes
+  def stable_sort!(elements, &block)
-    `git status --porcelain`.strip
+    elements.sort_by!.with_index { |element, index| [ block.call(element), index ] }
   end
 end

lib/kamal/utils/healthcheck_poller.rb

@@ -1,39 +0,0 @@
-class Kamal::Utils::HealthcheckPoller
-  TRAEFIK_HEALTHY_DELAY = 2
-
-  class HealthcheckError < StandardError; end
-
-  class << self
-    def wait_for_healthy(pause_after_ready: false, &block)
-      attempt = 1
-      max_attempts = KAMAL.config.healthcheck["max_attempts"]
-
-      begin
-        case status = block.call
-        when "healthy"
-          sleep TRAEFIK_HEALTHY_DELAY if pause_after_ready
-        when "running" # No health check configured
-          sleep KAMAL.config.readiness_delay if pause_after_ready
-        else
-          raise HealthcheckError, "container not ready (#{status})"
-        end
-      rescue HealthcheckError => e
-        if attempt <= max_attempts
-          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-          sleep attempt
-          attempt += 1
-          retry
-        else
-          raise
-        end
-      end
-
-      info "Container is healthy!"
-    end
-
-    private
-      def info(message)
-        SSHKit.config.output.info(message)
-      end
-  end
-end

lib/kamal/utils/sensitive.rb

@@ -1,4 +1,5 @@
 require "active_support/core_ext/module/delegation"
+require "sshkit"
 
 class Kamal::Utils::Sensitive
   # So SSHKit knows to redact these values.

lib/kamal/version.rb

@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "0.16.1"
+  VERSION = "1.5.0"
 end

test/cli/accessory_test.rb

@@ -7,7 +7,7 @@ class CliAccessoryTest < CliTestCase
 
     run_command("boot", "mysql").tap do |output|
       assert_match /docker login.*on 1.1.1.3/, output
-      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
     end
   end
 
@@ -21,9 +21,9 @@ class CliAccessoryTest < CliTestCase
       assert_match /docker login.*on 1.1.1.3/, output
       assert_match /docker login.*on 1.1.1.1/, output
       assert_match /docker login.*on 1.1.1.2/, output
-      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
-      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
-      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
     end
   end
 
@@ -48,6 +48,18 @@ class CliAccessoryTest < CliTestCase
     run_command("reboot", "mysql")
   end
 
+  test "reboot all" do
+    Kamal::Commands::Registry.any_instance.expects(:login).times(3)
+    Kamal::Cli::Accessory.any_instance.expects(:stop).with("mysql")
+    Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("mysql")
+    Kamal::Cli::Accessory.any_instance.expects(:boot).with("mysql", login: false)
+    Kamal::Cli::Accessory.any_instance.expects(:stop).with("redis")
+    Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("redis")
+    Kamal::Cli::Accessory.any_instance.expects(:boot).with("redis", login: false)
+
+    run_command("reboot", "all")
+  end
+
   test "start" do
     assert_match "docker container start app-mysql", run_command("start", "mysql")
   end
@@ -67,6 +79,10 @@ class CliAccessoryTest < CliTestCase
     assert_match "docker ps --filter label=service=app-mysql", run_command("details", "mysql")
   end
 
+  test "details with non-existent accessory" do
+    assert_equal "No accessory by the name of 'hello' (options: mysql and redis)", stderred { run_command("details", "hello") }
+  end
+
   test "details with all" do
     run_command("details", "all").tap do |output|
       assert_match "docker ps --filter label=service=app-mysql", output
@@ -97,7 +113,7 @@ class CliAccessoryTest < CliTestCase
 
   test "logs with follow" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.3 'docker logs app-mysql --timestamps --tail 10 --follow 2>&1'")
+      .with("ssh -t root@1.1.1.3 -p 22 'docker logs app-mysql --timestamps --tail 10 --follow 2>&1'")
 
     assert_match "docker logs app-mysql --timestamps --tail 10 --follow 2>&1", run_command("logs", "mysql", "--follow")
   end
@@ -136,8 +152,32 @@ class CliAccessoryTest < CliTestCase
     assert_match "rm -rf app-mysql", run_command("remove_service_directory", "mysql")
   end
 
+  test "hosts param respected" do
+    Kamal::Cli::Accessory.any_instance.expects(:directories).with("redis")
+    Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis")
+
+    run_command("boot", "redis", "--hosts", "1.1.1.1").tap do |output|
+      assert_match /docker login.*on 1.1.1.1/, output
+      assert_no_match /docker login.*on 1.1.1.2/, output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
+      assert_no_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
+    end
+  end
+
+  test "hosts param intersected with configuration" do
+    Kamal::Cli::Accessory.any_instance.expects(:directories).with("redis")
+    Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis")
+
+    run_command("boot", "redis", "--hosts", "1.1.1.1,1.1.1.3").tap do |output|
+      assert_match /docker login.*on 1.1.1.1/, output
+      assert_no_match /docker login.*on 1.1.1.3/, output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
+      assert_no_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.3", output
+    end
+  end
+
   private
     def run_command(*command)
-      stdouted { Kamal::Cli::Accessory.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Accessory.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
     end
 end

test/cli/app_test.rb

@@ -11,10 +11,11 @@ class CliAppTest < CliTestCase
   end
 
   test "boot will rename if same version is already running" do
+    Object.any_instance.stubs(:sleep)
     run_command("details") # Preheat Kamal const
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
       .returns("12345678") # running version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -22,9 +23,17 @@ class CliAppTest < CliTestCase
       .returns("running") # health check
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
       .returns("123") # old version
 
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
+      .returns("cordfile") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("unhealthy") # old version unhealthy
+
     run_command("boot").tap do |output|
       assert_match /Renaming container .* to .* as already deployed on 1.1.1.1/, output # Rename
       assert_match /docker rename app-web-latest app-web-latest_replaced_[0-9a-f]{16}/, output
@@ -36,7 +45,7 @@ class CliAppTest < CliTestCase
   end
 
   test "boot uses group strategy when specified" do
-    Kamal::Cli::App.any_instance.stubs(:on).with("1.1.1.1").twice # acquire & release lock
+    Kamal::Cli::App.any_instance.stubs(:on).with("1.1.1.1").times(3) # ensure locks dir, acquire & release lock
     Kamal::Cli::App.any_instance.stubs(:on).with([ "1.1.1.1" ]) # tag container
 
     # Strategy is used when booting the containers
@@ -46,17 +55,43 @@ class CliAppTest < CliTestCase
   end
 
   test "boot errors leave lock in place" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
-
     Kamal::Cli::App.any_instance.expects(:using_version).raises(RuntimeError)
 
-    assert !KAMAL.holding_lock?
+    assert_not KAMAL.holding_lock?
     assert_raises(RuntimeError) do
       stderred { run_command("boot") }
     end
     assert KAMAL.holding_lock?
   end
 
+  test "boot with assets" do
+    Object.any_instance.stubs(:sleep)
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678") # running version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("123").twice # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
+      .returns("") # old version
+
+    run_command("boot", config: :with_assets).tap do |output|
+      assert_match "docker tag dhh/app:latest dhh/app:latest", output
+      assert_match "/usr/bin/env mkdir -p .kamal/assets/volumes/app-web-latest ; cp -rnT .kamal/assets/extracted/app-web-latest .kamal/assets/volumes/app-web-latest ; cp -rnT .kamal/assets/extracted/app-web-latest .kamal/assets/volumes/app-web-123 || true ; cp -rnT .kamal/assets/extracted/app-web-123 .kamal/assets/volumes/app-web-latest || true", output
+      assert_match "/usr/bin/env mkdir -p .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets 2> /dev/null || true && docker run --name app-web-assets --detach --rm dhh/app:latest sleep 1000000 && docker cp -L app-web-assets:/public/assets/. .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets", output
+      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} /, output
+      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+      assert_match "/usr/bin/env find .kamal/assets/extracted -maxdepth 1 -name 'app-web-*' ! -name app-web-latest -exec rm -rf \"{}\" + ; find .kamal/assets/volumes -maxdepth 1 -name 'app-web-*' ! -name app-web-latest -exec rm -rf \"{}\" +", output
+    end
+  end
+
   test "start" do
     run_command("start").tap do |output|
       assert_match "docker start app-web-999", output
@@ -65,14 +100,18 @@ class CliAppTest < CliTestCase
 
   test "stop" do
     run_command("stop").tap do |output|
-      assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop", output
+      assert_match "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop", output
     end
   end
 
   test "stale_containers" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
-      .returns("12345678\n87654321")
+      .returns("12345678\n87654321\n")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("12345678\n")
 
     run_command("stale_containers").tap do |output|
       assert_match /Detected stale container for role web with version 87654321/, output
@@ -82,7 +121,11 @@ class CliAppTest < CliTestCase
   test "stop stale_containers" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
-      .returns("12345678\n87654321")
+      .returns("12345678\n87654321\n")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("12345678\n")
 
     run_command("stale_containers", "--stop").tap do |output|
       assert_match /Stopping stale container for role web with version 87654321/, output
@@ -98,7 +141,7 @@ class CliAppTest < CliTestCase
 
   test "remove" do
     run_command("remove").tap do |output|
-      assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop")}/, output
+      assert_match /#{Regexp.escape("sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop")}/, output
       assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output
       assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output
     end
@@ -124,17 +167,36 @@ class CliAppTest < CliTestCase
 
   test "exec" do
     run_command("exec", "ruby -v").tap do |output|
-      assert_match "docker run --rm dhh/app:latest ruby -v", output
+      assert_match "docker run --rm --env-file .kamal/env/roles/app-web.env dhh/app:latest ruby -v", output
     end
   end
 
   test "exec with reuse" do
     run_command("exec", "--reuse", "ruby -v").tap do |output|
-      assert_match "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done", output # Get current version
+      assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output # Get current version
       assert_match "docker exec app-web-999 ruby -v", output
     end
   end
 
+  test "exec interactive" do
+    SSHKit::Backend::Abstract.any_instance.expects(:exec)
+      .with("ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --env-file .kamal/env/roles/app-web.env dhh/app:latest ruby -v'")
+    run_command("exec", "-i", "ruby -v").tap do |output|
+      assert_match "Get most recent version available as an image...", output
+      assert_match "Launching interactive command with version latest via SSH from new container on 1.1.1.1...", output
+    end
+  end
+
+  test "exec interactive with reuse" do
+    SSHKit::Backend::Abstract.any_instance.expects(:exec)
+      .with("ssh -t root@1.1.1.1 -p 22 'docker exec -it app-web-999 ruby -v'")
+    run_command("exec", "-i", "--reuse", "ruby -v").tap do |output|
+      assert_match "Get current version of running container...", output
+      assert_match "Running /usr/bin/env sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done on 1.1.1.1", output
+      assert_match "Launching interactive command with version 999 via SSH from existing container on 1.1.1.1...", output
+    end
+  end
+
   test "containers" do
     run_command("containers").tap do |output|
       assert_match "docker container ls --all --filter label=service=app", output
@@ -149,41 +211,47 @@ class CliAppTest < CliTestCase
 
   test "logs" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest| xargs docker logs --timestamps --tail 10 2>&1'")
+      .with("ssh -t root@1.1.1.1 'sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1| xargs docker logs --timestamps --tail 10 2>&1'")
 
-    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --tail 100 2>&1", run_command("logs")
+    assert_match "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --tail 100 2>&1", run_command("logs")
   end
 
   test "logs with follow" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
+      .with("ssh -t root@1.1.1.1 -p 22 'sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
 
-    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
+    assert_match "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
   end
 
   test "version" do
     run_command("version").tap do |output|
-      assert_match "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done", output
+      assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output
     end
   end
 
 
   test "version through main" do
-    stdouted { Kamal::Cli::Main.start(["app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1"]) }.tap do |output|
+    stdouted { Kamal::Cli::Main.start([ "app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1" ]) }.tap do |output|
-      assert_match "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done", output
+      assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output
     end
   end
 
   private
     def run_command(*command, config: :with_accessories)
-      stdouted { Kamal::Cli::App.start([*command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1"]) }
+      stdouted { Kamal::Cli::App.start([ *command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1" ]) }
     end
 
     def stub_running
+      Object.any_instance.stubs(:sleep)
+
       SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
 
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
         .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
         .returns("running") # health check
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+        .returns("unhealthy") # health check
     end
 end

test/cli/build_test.rb

@@ -12,20 +12,23 @@ class CliBuildTest < CliTestCase
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" }
 
-    run_command("push").tap do |output|
+    run_command("push", "--verbose").tap do |output|
       assert_hook_ran "pre-build", output, **hook_variables
       assert_match /docker --version && docker buildx version/, output
-      assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
+      assert_match /git archive -tar HEAD | docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile - as .*@localhost/, output
     end
   end
 
   test "push without builder" do
-    stub_locking
+    stub_setup
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, "--version", "&&", :docker, :buildx, "version")
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*args| args[0..1] == [:docker, :buildx] }
+      .with(:docker, :buildx, :create, "--use", "--name", "kamal-app-multiarch")
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |*args| p args[0..6]; args[0..6] == [ :git, :archive, "--format=tar", :HEAD, "|", :docker, :buildx ] }
       .raises(SSHKit::Command::Failed.new("no builder"))
       .then
       .returns(true)
@@ -36,7 +39,7 @@ class CliBuildTest < CliTestCase
   end
 
   test "push with no buildx plugin" do
-    stub_locking
+    stub_setup
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, "--version", "&&", :docker, :buildx, "version")
       .raises(SSHKit::Command::Failed.new("no buildx"))
@@ -48,15 +51,17 @@ class CliBuildTest < CliTestCase
   test "push pre-build hook failure" do
     fail_hook("pre-build")
 
-    assert_raises(Kamal::Cli::HookError) { run_command("push") }
+    error = assert_raises(Kamal::Cli::HookError) { run_command("push") }
+    assert_equal "Hook `pre-build` failed:\nfailed", error.message
 
-    assert @executions.none? { |args| args[0..2] == [:docker, :buildx, :build] }
+    assert @executions.none? { |args| args[0..2] == [ :docker, :buildx, :build ] }
   end
 
   test "pull" do
     run_command("pull").tap do |output|
       assert_match /docker image rm --force dhh\/app:999/, output
       assert_match /docker pull dhh\/app:999/, output
+      assert_match "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:999 | grep -x app || (echo \"Image dhh/app:999 is missing the 'service' label\" && exit 1)", output
     end
   end
 
@@ -66,8 +71,16 @@ class CliBuildTest < CliTestCase
     end
   end
 
+  test "create remote" do
+    run_command("create", fixture: :with_remote_builder).tap do |output|
+      assert_match "Running /usr/bin/env true on 1.1.1.5", output
+      assert_match "docker context create kamal-app-native-remote-amd64 --description 'kamal-app-native-remote amd64 native host' --docker 'host=ssh://app@1.1.1.5'", output
+      assert_match "docker buildx create --name kamal-app-native-remote kamal-app-native-remote-amd64 --platform linux/amd64", output
+    end
+  end
+
   test "create with error" do
-    stub_locking
+    stub_setup
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with { |arg| arg == :docker }
       .raises(SSHKit::Command::Failed.new("stderr=error"))
@@ -95,14 +108,14 @@ class CliBuildTest < CliTestCase
   end
 
   private
-    def run_command(*command)
+    def run_command(*command, fixture: :with_accessories)
-      stdouted { Kamal::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Build.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) }
     end
 
     def stub_dependency_checks
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
         .with(:docker, "--version", "&&", :docker, :buildx, "version")
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |*args| args[0..1] == [:docker, :buildx] }
+        .with { |*args| args[0..1] == [ :docker, :buildx ] }
     end
 end

test/cli/cli_test_case.rb

@@ -21,20 +21,24 @@ class CliTestCase < ActiveSupport::TestCase
       Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
 
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |*args| @executions << args; args != [".kamal/hooks/#{hook}"] }
+        .with { |*args| @executions << args; args != [ ".kamal/hooks/#{hook}" ] }
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
         .with { |*args| args.first == ".kamal/hooks/#{hook}" }
         .raises(SSHKit::Command::Failed.new("failed"))
     end
 
-    def stub_locking
+    def stub_setup
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :mkdir && arg2 == "kamal_lock-app" }
+        .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :rm && arg2 == "kamal_lock-app/details" }
+        .with { |arg1, arg2, arg3| arg1 == :mkdir && arg2 == "-p" && arg3 == ".kamal/locks" }
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |arg1, arg2| arg1 == :mkdir && arg2 == ".kamal/locks/app" }
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |arg1, arg2| arg1 == :rm && arg2 == ".kamal/locks/app/details" }
     end
 
-    def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: nil)
+    def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: false)
       performer = `whoami`.strip
 
       assert_match "Running the #{hook} hook...\n", output
@@ -48,7 +52,7 @@ class CliTestCase < ActiveSupport::TestCase
         KAMAL_HOSTS=\"#{hosts}\"\s
         KAMAL_COMMAND=\"#{command}\"\s
         #{"KAMAL_SUBCOMMAND=\\\"#{subcommand}\\\"\\s" if subcommand}
-        #{"KAMAL_RUNTIME=\\\"#{runtime}\\\"\\s" if runtime}
+        #{"KAMAL_RUNTIME=\\\"\\d+\\\"\\s" if runtime}
         ;\s/usr/bin/env\s\.kamal/hooks/#{hook} }x
 
       assert_match expected, output

test/cli/env_test.rb

@@ -0,0 +1,37 @@
+require_relative "cli_test_case"
+
+class CliEnvTest < CliTestCase
+  test "push" do
+    run_command("push").tap do |output|
+      assert_match "Running /usr/bin/env mkdir -p .kamal/env/roles on 1.1.1.1", output
+      assert_match "Running /usr/bin/env mkdir -p .kamal/env/traefik on 1.1.1.1", output
+      assert_match "Running /usr/bin/env mkdir -p .kamal/env/accessories on 1.1.1.1", output
+      assert_match "Running /usr/bin/env mkdir -p .kamal/env/roles on 1.1.1.1", output
+      assert_match "Running /usr/bin/env mkdir -p .kamal/env/traefik on 1.1.1.2", output
+      assert_match "Running /usr/bin/env mkdir -p .kamal/env/accessories on 1.1.1.1", output
+      assert_match ".kamal/env/roles/app-web.env", output
+      assert_match ".kamal/env/roles/app-workers.env", output
+      assert_match ".kamal/env/traefik/traefik.env", output
+      assert_match ".kamal/env/accessories/app-redis.env", output
+    end
+  end
+
+  test "delete" do
+    run_command("delete").tap do |output|
+      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-web.env on 1.1.1.1", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-web.env on 1.1.1.2", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-workers.env on 1.1.1.3", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-workers.env on 1.1.1.4", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/traefik/traefik.env on 1.1.1.1", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/traefik/traefik.env on 1.1.1.2", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-redis.env on 1.1.1.1", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-redis.env on 1.1.1.2", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-mysql.env on 1.1.1.3", output
+    end
+  end
+
+  private
+    def run_command(*command)
+      stdouted { Kamal::Cli::Env.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
+    end
+end

test/cli/healthcheck_test.rb

@@ -5,12 +5,13 @@ class CliHealthcheckTest < CliTestCase
     # Prevent expected failures from outputting to terminal
     Thread.report_on_exception = false
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
+      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--env-file", ".kamal/env/roles/app-web.env", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false)
 
@@ -34,12 +35,12 @@ class CliHealthcheckTest < CliTestCase
     # Prevent expected failures from outputting to terminal
     Thread.report_on_exception = false
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
+      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--env-file", ".kamal/env/roles/app-web.env", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false)
 
@@ -64,8 +65,18 @@ class CliHealthcheckTest < CliTestCase
     assert_match "container not ready (unhealthy)", exception.message
   end
 
+  test "raises an exception if primary does not have traefik" do
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).never
+
+    exception = assert_raises do
+      run_command("perform", config_file: "test/fixtures/deploy_workers_only.yml")
+    end
+
+    assert_equal "The primary host is not configured to run Traefik", exception.message
+  end
+
   private
-    def run_command(*command)
+    def run_command(*command, config_file: "test/fixtures/deploy_with_accessories.yml")
-      stdouted { Kamal::Cli::Healthcheck.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Healthcheck.start([ *command, "-c", config_file ]) }
     end
 end

test/cli/lock_test.rb

@@ -2,19 +2,19 @@ require_relative "cli_test_case"
 
 class CliLockTest < CliTestCase
   test "status" do
-    run_command("status") do |output|
+    run_command("status").tap do |output|
-      assert_match "stat lock", output
+      assert_match "Running /usr/bin/env stat .kamal/locks/app > /dev/null && cat .kamal/locks/app/details | base64 -d on 1.1.1.1", output
     end
   end
 
   test "release" do
-    run_command("release") do |output|
+    run_command("release").tap do |output|
-      assert_match "rm -rf lock", output
+      assert_match "Released the deploy lock", output
     end
   end
 
   private
     def run_command(*command)
-      stdouted { Kamal::Cli::Lock.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Lock.start([ *command, "-v", "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
     end
 end

test/cli/main_test.rb

@@ -2,28 +2,64 @@ require_relative "cli_test_case"
 
 class CliMainTest < CliTestCase
   test "setup" do
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap")
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ])
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:deploy)
 
-    run_command("setup")
+    run_command("setup").tap do |output|
+      assert_match /Ensure Docker is installed.../, output
+      assert_match /Push env files.../, output
+    end
+  end
+
+  test "setup with skip_push" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
+    # deploy
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
+
+    run_command("setup", "--skip_push").tap do |output|
+      assert_match /Ensure Docker is installed.../, output
+      assert_match /Push env files.../, output
+      # deploy
+      assert_match /Acquiring the deploy lock/, output
+      assert_match /Log into image registry/, output
+      assert_match /Pull app image/, output
+      assert_match /Ensure Traefik is running/, output
+      assert_match /Ensure app can pass healthcheck/, output
+      assert_match /Detect stale containers/, output
+      assert_match /Prune old containers and images/, output
+      assert_match /Releasing the deploy lock/, output
+    end
   end
 
   test "deploy" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "deploy" }
 
-    run_command("deploy").tap do |output|
+    run_command("deploy", "--verbose").tap do |output|
       assert_hook_ran "pre-connect", output, **hook_variables
       assert_match /Log into image registry/, output
       assert_match /Build and push app image/, output
@@ -32,7 +68,7 @@ class CliMainTest < CliTestCase
       assert_match /Ensure app can pass healthcheck/, output
       assert_match /Detect stale containers/, output
       assert_match /Prune old containers and images/, output
-      assert_hook_ran "post-deploy", output, **hook_variables, runtime: 0
+      assert_hook_ran "post-deploy", output, **hook_variables, runtime: true
     end
   end
 
@@ -43,7 +79,7 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
@@ -63,11 +99,17 @@ class CliMainTest < CliTestCase
     Thread.report_on_exception = false
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*arg| arg[0..1] == [:mkdir, 'kamal_lock-app'] }
+      .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }
-      .raises(RuntimeError, "mkdir: cannot create directory ‘kamal_lock-app’: File exists")
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |*args| args == [ :mkdir, "-p", ".kamal/locks" ] }
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |*arg| arg[0..1] == [ :mkdir, ".kamal/locks/app" ] }
+      .raises(RuntimeError, "mkdir: cannot create directory ‘kamal/locks/app’: File exists")
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_debug)
-      .with(:stat, 'kamal_lock-app', ">", "/dev/null", "&&", :cat, "kamal_lock-app/details", "|", :base64, "-d")
+      .with(:stat, ".kamal/locks/app", ">", "/dev/null", "&&", :cat, ".kamal/locks/app/details", "|", :base64, "-d")
 
     assert_raises(Kamal::Cli::LockError) do
       run_command("deploy")
@@ -78,7 +120,13 @@ class CliMainTest < CliTestCase
     Thread.report_on_exception = false
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*arg| arg[0..1] == [:mkdir, 'kamal_lock-app'] }
+      .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |*args| args == [ :mkdir, "-p", ".kamal/locks" ] }
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |*arg| arg[0..1] == [ :mkdir, ".kamal/locks/app" ] }
       .raises(SocketError, "getaddrinfo: nodename nor servname provided, or not known")
 
     assert_raises(SSHKit::Runner::ExecuteError) do
@@ -93,11 +141,11 @@ class CliMainTest < CliTestCase
       .with("kamal:cli:registry:login", [], invoke_options)
       .raises(RuntimeError)
 
-    assert !KAMAL.holding_lock?
+    assert_not KAMAL.holding_lock?
     assert_raises(RuntimeError) do
       stderred { run_command("deploy") }
     end
-    assert !KAMAL.holding_lock?
+    assert_not KAMAL.holding_lock?
   end
 
   test "deploy with skipped hooks" do
@@ -107,40 +155,63 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
     run_command("deploy", "--skip_hooks") do
-      refute_match /Running the post-deploy hook.../, output
+      assert_no_match /Running the post-deploy hook.../, output
     end
   end
 
+  test "deploy without healthcheck if primary host doesn't have traefik" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_workers_only.yml", "version" => "999", "skip_hooks" => false }
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options).never
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
+
+    run_command("deploy", config_file: "deploy_workers_only")
+  end
+
   test "deploy with missing secrets" do
-    assert_raises(KeyError) do
+    invoke_options = { "config_file" => "test/fixtures/deploy_with_secrets.yml", "version" => "999", "skip_hooks" => false }
-      run_command("deploy", config_file: "deploy_with_secrets")
+
-    end
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
+
+    run_command("deploy", config_file: "deploy_with_secrets")
   end
 
   test "redeploy" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
 
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
 
     hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "redeploy" }
 
-    run_command("redeploy").tap do |output|
+    run_command("redeploy", "--verbose").tap do |output|
       assert_hook_ran "pre-connect", output, **hook_variables
       assert_match /Build and push app image/, output
       assert_hook_ran "pre-deploy", output, **hook_variables
       assert_match /Running the pre-deploy hook.../, output
       assert_match /Ensure app can pass healthcheck/, output
-      assert_hook_ran "post-deploy", output, **hook_variables, runtime: "0"
+      assert_hook_ran "post-deploy", output, **hook_variables, runtime: true
     end
   end
 
@@ -149,7 +220,7 @@ class CliMainTest < CliTestCase
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
 
     run_command("redeploy", "--skip_push").tap do |output|
@@ -170,52 +241,59 @@ class CliMainTest < CliTestCase
   end
 
   test "rollback good version" do
+    Object.any_instance.stubs(:sleep)
     [ "web", "workers" ].each do |role|
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :container, :ls, "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false)
+        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false)
         .returns("").at_least_once
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
         .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet")
         .returns("version-to-rollback\n").at_least_once
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-#{role}-}; done", raise_on_non_zero_exit: false)
+        .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=#{role} --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=#{role} --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-#{role}-}; done", raise_on_non_zero_exit: false)
         .returns("version-to-rollback\n").at_least_once
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
         .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
         .returns("running").at_least_once # health check
     end
 
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
+      .returns("corddirectory").at_least_once # health check
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-version-to-rollback$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("unhealthy").at_least_once # health check
+
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     hook_variables = { version: 123, service_version: "app@123", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "rollback" }
 
-    run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
+    run_command("rollback", "--verbose", "123", config_file: "deploy_with_accessories").tap do |output|
-      assert_match "Start container with version 123", output
       assert_hook_ran "pre-deploy", output, **hook_variables
       assert_match "docker tag dhh/app:123 dhh/app:latest", output
-      assert_match "docker start app-web-123", output
+      assert_match "docker run --detach --restart unless-stopped --name app-web-123", output
       assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running"
-      assert_hook_ran "post-deploy", output, **hook_variables, runtime: "0"
+      assert_hook_ran "post-deploy", output, **hook_variables, runtime: true
     end
   end
 
   test "rollback without old version" do
     Kamal::Cli::Main.any_instance.stubs(:container_available?).returns(true)
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep)
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep)
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
       .returns("").at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
       .returns("").at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
       .returns("running").at_least_once # health check
 
     run_command("rollback", "123").tap do |output|
-      assert_match "Start container with version 123", output
+      assert_match "docker run --detach --restart unless-stopped --name app-web-123", output
-      assert_match "docker start app-web-123 || docker run --detach --restart unless-stopped --name app-web-123", output
       assert_no_match "docker stop", output
     end
   end
@@ -230,7 +308,7 @@ class CliMainTest < CliTestCase
 
   test "audit" do
     run_command("audit").tap do |output|
-      assert_match /tail -n 50 kamal-app-audit.log on 1.1.1.1/, output
+      assert_match %r{tail -n 50 \.kamal/app-audit.log on 1.1.1.1}, output
       assert_match /App Host: 1.1.1.1/, output
     end
   end
@@ -239,8 +317,8 @@ class CliMainTest < CliTestCase
     run_command("config", config_file: "deploy_simple").tap do |output|
       config = YAML.load(output)
 
-      assert_equal ["web"], config[:roles]
+      assert_equal [ "web" ], config[:roles]
-      assert_equal ["1.1.1.1", "1.1.1.2"], config[:hosts]
+      assert_equal [ "1.1.1.1", "1.1.1.2" ], config[:hosts]
       assert_equal "999", config[:version]
       assert_equal "dhh/app", config[:repository]
       assert_equal "dhh/app:999", config[:absolute_image]
@@ -252,8 +330,8 @@ class CliMainTest < CliTestCase
     run_command("config", config_file: "deploy_with_roles").tap do |output|
       config = YAML.load(output)
 
-      assert_equal ["web", "workers"], config[:roles]
+      assert_equal [ "web", "workers" ], config[:roles]
-      assert_equal ["1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4"], config[:hosts]
+      assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], config[:hosts]
       assert_equal "999", config[:version]
       assert_equal "registry.digitalocean.com/dhh/app", config[:repository]
       assert_equal "registry.digitalocean.com/dhh/app:999", config[:absolute_image]
@@ -261,12 +339,35 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "config with primary web role override" do
+    run_command("config", config_file: "deploy_primary_web_role_override").tap do |output|
+      config = YAML.load(output)
+
+      assert_equal [ "web_chicago", "web_tokyo" ], config[:roles]
+      assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], config[:hosts]
+      assert_equal "1.1.1.3", config[:primary_host]
+    end
+  end
+
   test "config with destination" do
     run_command("config", "-d", "world", config_file: "deploy_for_dest").tap do |output|
       config = YAML.load(output)
 
-      assert_equal ["web"], config[:roles]
+      assert_equal [ "web" ], config[:roles]
-      assert_equal ["1.1.1.1", "1.1.1.2"], config[:hosts]
+      assert_equal [ "1.1.1.1", "1.1.1.2" ], config[:hosts]
+      assert_equal "999", config[:version]
+      assert_equal "registry.digitalocean.com/dhh/app", config[:repository]
+      assert_equal "registry.digitalocean.com/dhh/app:999", config[:absolute_image]
+      assert_equal "app-999", config[:service_with_version]
+    end
+  end
+
+  test "config with aliases" do
+    run_command("config", config_file: "deploy_with_aliases").tap do |output|
+      config = YAML.load(output)
+
+      assert_equal [ "web", "web_tokyo", "workers", "workers_tokyo" ], config[:roles]
+      assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], config[:hosts]
       assert_equal "999", config[:version]
       assert_equal "registry.digitalocean.com/dhh/app", config[:repository]
       assert_equal "registry.digitalocean.com/dhh/app:999", config[:absolute_image]
@@ -332,11 +433,33 @@ class CliMainTest < CliTestCase
     run_command("envify")
   end
 
-  test "envify with destination" do
+  test "envify with blank line trimming" do
-    File.expects(:read).with(".env.staging.erb").returns("HELLO=<%= 'world' %>")
+    file = <<~EOF
-    File.expects(:write).with(".env.staging", "HELLO=world", perm: 0600)
+      HELLO=<%= 'world' %>
+      <% if true -%>
+      KEY=value
+      <% end  -%>
+    EOF
 
-    run_command("envify", "-d", "staging")
+    File.expects(:read).with(".env.erb").returns(file.strip)
+    File.expects(:write).with(".env", "HELLO=world\nKEY=value\n", perm: 0600)
+
+    run_command("envify")
+  end
+
+  test "envify with destination" do
+    File.expects(:read).with(".env.world.erb").returns("HELLO=<%= 'world' %>")
+    File.expects(:write).with(".env.world", "HELLO=world", perm: 0600)
+
+    run_command("envify", "-d", "world", config_file: "deploy_for_dest")
+  end
+
+  test "envify with skip_push" do
+    File.expects(:read).with(".env.erb").returns("HELLO=<%= 'world' %>")
+    File.expects(:write).with(".env", "HELLO=world", perm: 0600)
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push").never
+    run_command("envify", "--skip-push")
   end
 
   test "remove with confirmation" do
@@ -370,6 +493,6 @@ class CliMainTest < CliTestCase
 
   private
     def run_command(*command, config_file: "deploy_simple")
-      stdouted { Kamal::Cli::Main.start([*command, "-c", "test/fixtures/#{config_file}.yml"]) }
+      stdouted { Kamal::Cli::Main.start([ *command, "-c", "test/fixtures/#{config_file}.yml" ]) }
     end
 end

test/cli/prune_test.rb

@@ -10,7 +10,7 @@ class CliPruneTest < CliTestCase
 
   test "images" do
     run_command("images").tap do |output|
-      assert_match "docker image prune --force --filter label=service=app --filter dangling=true on 1.1.1.", output
+      assert_match "docker image prune --force --filter label=service=app on 1.1.1.", output
       assert_match "docker image ls --filter label=service=app --format '{{.ID}} {{.Repository}}:{{.Tag}}' | grep -v -w \"$(docker container ls -a --format '{{.Image}}\\|' --filter label=service=app | tr -d '\\n')dhh/app:latest\\|dhh/app:<none>\" | while read image tag; do docker rmi $tag; done on 1.1.1.", output
     end
   end
@@ -18,11 +18,21 @@ class CliPruneTest < CliTestCase
   test "containers" do
     run_command("containers").tap do |output|
       assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output
+      assert_match /docker container prune --force --filter label=service=healthcheck-app on 1.1.1.\d/, output
+     end
+
+    run_command("containers", "--retain", "10").tap do |output|
+      assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +11 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output
+      assert_match /docker container prune --force --filter label=service=healthcheck-app on 1.1.1.\d/, output
+    end
+
+    assert_raises(RuntimeError, "retain must be at least 1") do
+      run_command("containers", "--retain", "0")
     end
   end
 
   private
     def run_command(*command)
-      stdouted { Kamal::Cli::Prune.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Prune.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
     end
 end

test/cli/registry_test.rb

@@ -16,6 +16,6 @@ class CliRegistryTest < CliTestCase
 
   private
     def run_command(*command)
-      stdouted { Kamal::Cli::Registry.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Registry.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
     end
 end

test/cli/server_test.rb

@@ -3,13 +3,15 @@ require_relative "cli_test_case"
 class CliServerTest < CliTestCase
   test "bootstrap already installed" do
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(true).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
 
     assert_equal "", run_command("bootstrap")
   end
 
   test "bootstrap install as non-root user" do
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ]', raise_on_non_zero_exit: false).returns(false).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', raise_on_non_zero_exit: false).returns(false).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
 
     assert_raise RuntimeError, "Docker is not installed on 1.1.1.1, 1.1.1.3, 1.1.1.4, 1.1.1.2 and can't be automatically installed without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/" do
       run_command("bootstrap")
@@ -18,18 +20,22 @@ class CliServerTest < CliTestCase
 
   test "bootstrap install as root user" do
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ]', raise_on_non_zero_exit: false).returns(true).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', raise_on_non_zero_exit: false).returns(true).at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:curl, "-fsSL", "https://get.docker.com", "|", :sh).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:sh, "-c", "'curl -fsSL https://get.docker.com || wget -O - https://get.docker.com || echo \"exit 1\"'", "|", :sh).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
+    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(".kamal/hooks/docker-setup", anything).at_least_once
 
     run_command("bootstrap").tap do |output|
       ("1.1.1.1".."1.1.1.4").map do |host|
         assert_match "Missing Docker on #{host}. Installing…", output
+        assert_match "Running the docker-setup hook", output
       end
     end
   end
 
   private
     def run_command(*command)
-      stdouted { Kamal::Cli::Server.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Server.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
     end
 end

test/cli/traefik_test.rb

@@ -4,23 +4,25 @@ class CliTraefikTest < CliTestCase
   test "boot" do
     run_command("boot").tap do |output|
       assert_match "docker login", output
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
+      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
     end
   end
 
   test "reboot" do
     Kamal::Commands::Registry.any_instance.expects(:login).twice
 
-    run_command("reboot").tap do |output|
+    run_command("reboot", "-y").tap do |output|
       assert_match "docker container stop traefik", output
       assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik", output
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
+      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
     end
   end
 
   test "reboot --rolling" do
-    run_command("reboot", "--rolling").tap do |output|
+    Object.any_instance.stubs(:sleep)
-      assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.1", output.lines[3]
+
+    run_command("reboot", "--rolling", "-y").tap do |output|
+      assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.1", output
     end
   end
 
@@ -62,7 +64,7 @@ class CliTraefikTest < CliTestCase
 
   test "logs with follow" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 'docker logs traefik --timestamps --tail 10 --follow 2>&1'")
+      .with("ssh -t root@1.1.1.1 -p 22 'docker logs traefik --timestamps --tail 10 --follow 2>&1'")
 
     assert_match "docker logs traefik --timestamps --tail 10 --follow", run_command("logs", "--follow")
   end
@@ -89,6 +91,6 @@ class CliTraefikTest < CliTestCase
 
   private
     def run_command(*command)
-      stdouted { Kamal::Cli::Traefik.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Traefik.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
     end
 end

test/commander_test.rb

@@ -14,6 +14,20 @@ class CommanderTest < ActiveSupport::TestCase
 
     @kamal.specific_hosts = [ "1.1.1.1", "1.1.1.2" ]
     assert_equal [ "1.1.1.1", "1.1.1.2" ], @kamal.hosts
+
+    @kamal.specific_hosts = [ "1.1.1.1*" ]
+    assert_equal [ "1.1.1.1" ], @kamal.hosts
+
+    @kamal.specific_hosts = [ "1.1.1.*", "*.1.2.*" ]
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @kamal.hosts
+
+    @kamal.specific_hosts = [ "*" ]
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @kamal.hosts
+
+    exception = assert_raises(ArgumentError) do
+      @kamal.specific_hosts = [ "*miss" ]
+    end
+    assert_match /hosts match for \*miss/, exception.message
   end
 
   test "filtering hosts by filtering roles" do
@@ -21,6 +35,11 @@ class CommanderTest < ActiveSupport::TestCase
 
     @kamal.specific_roles = [ "web" ]
     assert_equal [ "1.1.1.1", "1.1.1.2" ], @kamal.hosts
+
+    exception = assert_raises(ArgumentError) do
+      @kamal.specific_roles = [ "*miss" ]
+    end
+    assert_match /roles match for \*miss/, exception.message
   end
 
   test "filtering roles" do
@@ -28,6 +47,20 @@ class CommanderTest < ActiveSupport::TestCase
 
     @kamal.specific_roles = [ "workers" ]
     assert_equal [ "workers" ], @kamal.roles.map(&:name)
+
+    @kamal.specific_roles = [ "w*" ]
+    assert_equal [ "web", "workers" ], @kamal.roles.map(&:name)
+
+    @kamal.specific_roles = [ "we*", "*orkers" ]
+    assert_equal [ "web", "workers" ], @kamal.roles.map(&:name)
+
+    @kamal.specific_roles = [ "*" ]
+    assert_equal [ "web", "workers" ], @kamal.roles.map(&:name)
+
+    exception = assert_raises(ArgumentError) do
+      @kamal.specific_roles = [ "*miss" ]
+    end
+    assert_match /roles match for \*miss/, exception.message
   end
 
   test "filtering roles by filtering hosts" do
@@ -49,9 +82,15 @@ class CommanderTest < ActiveSupport::TestCase
     assert_equal "1.1.1.3", @kamal.primary_host
   end
 
+  test "primary_role" do
+    assert_equal "web", @kamal.primary_role.name
+    @kamal.specific_roles = "workers"
+    assert_equal "workers", @kamal.primary_role.name
+  end
+
   test "roles_on" do
-    assert_equal [ "web" ], @kamal.roles_on("1.1.1.1")
+    assert_equal [ "web" ], @kamal.roles_on("1.1.1.1").map(&:name)
-    assert_equal [ "workers" ], @kamal.roles_on("1.1.1.3")
+    assert_equal [ "workers" ], @kamal.roles_on("1.1.1.3").map(&:name)
   end
 
   test "default group strategy" do
@@ -70,6 +109,22 @@ class CommanderTest < ActiveSupport::TestCase
     assert_equal({ in: :groups, limit: 1, wait: 2 }, @kamal.boot_strategy)
   end
 
+  test "percentage-based group strategy limit is at least 1" do
+    configure_with(:deploy_with_low_percentage_boot_strategy)
+
+    assert_equal({ in: :groups, limit: 1, wait: 2 }, @kamal.boot_strategy)
+  end
+
+  test "try to match the primary role from a list of specific roles" do
+    configure_with(:deploy_primary_web_role_override)
+
+    @kamal.specific_roles = [ "web_*" ]
+    assert_equal [ "web_tokyo", "web_chicago" ], @kamal.roles.map(&:name)
+    assert_equal "web_tokyo", @kamal.primary_role.name
+    assert_equal "1.1.1.3", @kamal.primary_host
+    assert_equal [ "1.1.1.3", "1.1.1.4", "1.1.1.1", "1.1.1.2" ], @kamal.hosts
+  end
+
   private
     def configure_with(variant)
       @kamal = Kamal::Commander.new.tap do |kamal|

test/commands/accessory_test.rb

@@ -34,6 +34,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
           ]
         },
         "busybox" => {
+          "service" => "custom-busybox",
           "image" => "busybox:latest",
           "host" => "1.1.1.7"
         }
@@ -49,15 +50,15 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 -e MYSQL_ROOT_PASSWORD=\"secret123\" -e MYSQL_ROOT_HOST=\"%\" --label service=\"app-mysql\" private.registry/mysql:8.0",
+      "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --label service=\"app-mysql\" private.registry/mysql:8.0",
       new_command(:mysql).run.join(" ")
 
     assert_equal \
-      "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 -e SOMETHING=\"else\" --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest",
+      "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --env SOMETHING=\"else\" --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest",
       new_command(:redis).run.join(" ")
 
     assert_equal \
-      "docker run --name app-busybox --detach --restart unless-stopped --log-opt max-size=\"10m\" --label service=\"app-busybox\" busybox:latest",
+      "docker run --name custom-busybox --detach --restart unless-stopped --log-opt max-size=\"10m\" --env-file .kamal/env/accessories/custom-busybox.env --label service=\"custom-busybox\" busybox:latest",
       new_command(:busybox).run.join(" ")
   end
 
@@ -65,7 +66,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --name app-busybox --detach --restart unless-stopped --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app-busybox\" busybox:latest",
+      "docker run --name custom-busybox --detach --restart unless-stopped --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --env-file .kamal/env/accessories/custom-busybox.env --label service=\"custom-busybox\" busybox:latest",
       new_command(:busybox).run.join(" ")
   end
 
@@ -90,7 +91,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "execute in new container" do
     assert_equal \
-      "docker run --rm -e MYSQL_ROOT_PASSWORD=\"secret123\" -e MYSQL_ROOT_HOST=\"%\" private.registry/mysql:8.0 mysql -u root",
+      "docker run --rm --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" private.registry/mysql:8.0 mysql -u root",
       new_command(:mysql).execute_in_new_container("mysql", "-u", "root").join(" ")
   end
 
@@ -102,14 +103,14 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "execute in new container over ssh" do
     new_command(:mysql).stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do
-      assert_match %r|docker run -it --rm -e MYSQL_ROOT_PASSWORD=\"secret123\" -e MYSQL_ROOT_HOST=\"%\" private.registry/mysql:8.0 mysql -u root|,
+      assert_match %r{docker run -it --rm --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" private.registry/mysql:8.0 mysql -u root},
         new_command(:mysql).execute_in_new_container_over_ssh("mysql", "-u", "root")
     end
   end
 
   test "execute in existing container over ssh" do
     new_command(:mysql).stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do
-      assert_match %r|docker exec -it app-mysql mysql -u root|,
+      assert_match %r{docker exec -it app-mysql mysql -u root},
         new_command(:mysql).execute_in_existing_container_over_ssh("mysql", "-u", "root")
     end
   end
@@ -128,7 +129,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "follow logs" do
     assert_equal \
-      "ssh -t root@1.1.1.5 'docker logs app-mysql --timestamps --tail 10 --follow 2>&1'",
+      "ssh -t root@1.1.1.5 -p 22 'docker logs app-mysql --timestamps --tail 10 --follow 2>&1'",
       new_command(:mysql).follow_logs
   end
 
@@ -144,6 +145,14 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
       new_command(:mysql).remove_image.join(" ")
   end
 
+  test "make_env_directory" do
+    assert_equal "mkdir -p .kamal/env/accessories", new_command(:mysql).make_env_directory.join(" ")
+  end
+
+  test "remove_env_file" do
+    assert_equal "rm -f .kamal/env/accessories/app-mysql.env", new_command(:mysql).remove_env_file.join(" ")
+  end
+
   private
     def new_command(accessory)
       Kamal::Commands::Accessory.new(Kamal::Configuration.new(@config), name: accessory)

test/commands/app_test.rb

@@ -3,6 +3,7 @@ require "test_helper"
 class CommandsAppTest < ActiveSupport::TestCase
   setup do
     ENV["RAILS_MASTER_KEY"] = "456"
+    Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
 
     @config = { service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], env: { "secret" => [ "RAILS_MASTER_KEY" ] } }
   end
@@ -13,21 +14,21 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with hostname" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run(hostname: "myhost").join(" ")
   end
 
   test "run with volumes" do
-    @config[:volumes] = ["/local/path:/container/path" ]
+    @config[:volumes] = [ "/local/path:/container/path" ]
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -35,7 +36,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "path" => "/healthz" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/healthz || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -43,7 +44,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "cmd" => "/bin/up" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"/bin/up\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -51,14 +52,14 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"/bin/healthy\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with custom options" do
     @config[:servers] = { "web" => [ "1.1.1.1" ], "jobs" => { "hosts" => [ "1.1.1.2" ], "cmd" => "bin/jobs", "options" => { "mount" => "somewhere", "cap-add" => true } } }
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
+      "docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-jobs.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --label destination --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
       new_command(role: "jobs").run.join(" ")
   end
 
@@ -66,7 +67,16 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      new_command.run.join(" ")
+  end
+
+  test "run with role logging config" do
+    @config[:logging] = { "driver" => "local", "options" => { "max-size" => "10m", "max-file" => "3" } }
+    @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "logging" => { "driver" => "local", "options" => { "max-size" => "100m" } } } }
+
+    assert_equal \
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -83,28 +93,16 @@ class CommandsAppTest < ActiveSupport::TestCase
       new_command.start.join(" ")
   end
 
-  test "start_or_run" do
-    assert_equal \
-      "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
-      new_command.start_or_run.join(" ")
-  end
-
-  test "start_or_run with hostname" do
-    assert_equal \
-      "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
-      new_command.start_or_run(hostname: "myhost").join(" ")
-  end
-
   test "stop" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop",
       new_command.stop.join(" ")
   end
 
   test "stop with custom stop wait time" do
     @config[:stop_wait_time] = 30
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop -t 30",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop -t 30",
       new_command.stop.join(" ")
   end
 
@@ -130,115 +128,140 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "logs" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs 2>&1",
       new_command.logs.join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --since 5m 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --since 5m 2>&1",
       new_command.logs(since: "5m").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --tail 100 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --tail 100 2>&1",
       new_command.logs(lines: "100").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --since 5m --tail 100 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --since 5m --tail 100 2>&1",
       new_command.logs(since: "5m", lines: "100").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs 2>&1 | grep 'my-id'",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs 2>&1 | grep 'my-id'",
       new_command.logs(grep: "my-id").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
       new_command.logs(since: "5m", grep: "my-id").join(" ")
   end
 
   test "follow logs" do
-    assert_match \
+    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1",
+      "ssh -t root@app-1 -p 22 'sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --timestamps --follow 2>&1'",
       new_command.follow_logs(host: "app-1")
 
-    assert_match \
+    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1 | grep \"Completed\"",
+      "ssh -t root@app-1 -p 22 'sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --timestamps --follow 2>&1 | grep \"Completed\"'",
       new_command.follow_logs(host: "app-1", grep: "Completed")
+
+    assert_equal \
+      "ssh -t root@app-1 -p 22 'sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --timestamps --tail 123 --follow 2>&1'",
+      new_command.follow_logs(host: "app-1", lines: 123)
+
+    assert_equal \
+      "ssh -t root@app-1 -p 22 'sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --timestamps --tail 123 --follow 2>&1 | grep \"Completed\"'",
+      new_command.follow_logs(host: "app-1", lines: 123, grep: "Completed")
   end
 
 
   test "execute in new container" do
     assert_equal \
-      "docker run --rm -e RAILS_MASTER_KEY=\"456\" dhh/app:999 bin/rails db:setup",
+      "docker run --rm --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails db:setup",
-      new_command.execute_in_new_container("bin/rails", "db:setup").join(" ")
+      new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ")
+  end
+
+  test "execute in new container with env" do
+    assert_equal \
+      "docker run --rm --env-file .kamal/env/roles/app-web.env --env foo=\"bar\" dhh/app:999 bin/rails db:setup",
+      new_command.execute_in_new_container("bin/rails", "db:setup", env: { "foo" => "bar" }).join(" ")
   end
 
   test "execute in new container with custom options" do
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } }
     assert_equal \
-      "docker run --rm -e RAILS_MASTER_KEY=\"456\" --mount \"somewhere\" --cap-add dhh/app:999 bin/rails db:setup",
+      "docker run --rm --env-file .kamal/env/roles/app-web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails db:setup",
-      new_command.execute_in_new_container("bin/rails", "db:setup").join(" ")
+      new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ")
   end
 
   test "execute in existing container" do
     assert_equal \
       "docker exec app-web-999 bin/rails db:setup",
-      new_command.execute_in_existing_container("bin/rails", "db:setup").join(" ")
+      new_command.execute_in_existing_container("bin/rails", "db:setup", env: {}).join(" ")
+  end
+
+  test "execute in existing container with env" do
+    assert_equal \
+      "docker exec --env foo=\"bar\" app-web-999 bin/rails db:setup",
+      new_command.execute_in_existing_container("bin/rails", "db:setup", env: { "foo" => "bar" }).join(" ")
   end
 
   test "execute in new container over ssh" do
-    assert_match %r|docker run -it --rm -e RAILS_MASTER_KEY=\"456\" dhh/app:999 bin/rails c|,
+    assert_match %r{docker run -it --rm --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails c},
-      new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1")
+      new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1", env: {})
   end
 
   test "execute in new container with custom options over ssh" do
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } }
-    assert_match %r|docker run -it --rm -e RAILS_MASTER_KEY=\"456\" --mount \"somewhere\" --cap-add dhh/app:999 bin/rails c|,
+    assert_match %r{docker run -it --rm --env-file .kamal/env/roles/app-web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails c},
-      new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1")
+      new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1", env: {})
   end
 
   test "execute in existing container over ssh" do
-    assert_match %r|docker exec -it app-web-999 bin/rails c|,
+    assert_match %r{docker exec -it app-web-999 bin/rails c},
-      new_command.execute_in_existing_container_over_ssh("bin/rails", "c", host: "app-1")
+      new_command.execute_in_existing_container_over_ssh("bin/rails", "c", host: "app-1", env: {})
   end
 
   test "run over ssh" do
-    assert_equal "ssh -t root@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with custom user" do
     @config[:ssh] = { "user" => "app" }
-    assert_equal "ssh -t app@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -t app@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+  end
+
+  test "run over ssh with custom port" do
+    @config[:ssh] = { "port" => "2222" }
+    assert_equal "ssh -t root@1.1.1.1 -p 2222 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with proxy" do
     @config[:ssh] = { "proxy" => "2.2.2.2" }
-    assert_equal "ssh -J root@2.2.2.2 -t root@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -J root@2.2.2.2 -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with proxy user" do
     @config[:ssh] = { "proxy" => "app@2.2.2.2" }
-    assert_equal "ssh -J app@2.2.2.2 -t root@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -J app@2.2.2.2 -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with custom user with proxy" do
     @config[:ssh] = { "user" => "app", "proxy" => "2.2.2.2" }
-    assert_equal "ssh -J root@2.2.2.2 -t app@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -J root@2.2.2.2 -t app@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "run over ssh with proxy_command" do
     @config[:ssh] = { "proxy_command" => "ssh -W %h:%p user@proxy-server" }
-    assert_equal "ssh -o ProxyCommand='ssh -W %h:%p user@proxy-server' -t root@1.1.1.1 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
+    assert_equal "ssh -o ProxyCommand='ssh -W %h:%p user@proxy-server' -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1")
   end
 
   test "current_running_container_id" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest",
+    "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1",
       new_command.current_running_container_id.join(" ")
   end
 
   test "current_running_container_id with destination" do
     @destination = "staging"
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --filter status=restarting --latest",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest-staging --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --filter status=restarting' | head -1",
       new_command.current_running_container_id.join(" ")
   end
 
@@ -250,7 +273,7 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "current_running_version" do
     assert_equal \
-      "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done",
+      "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done",
       new_command.current_running_version.join(" ")
   end
 
@@ -328,14 +351,75 @@ class CommandsAppTest < ActiveSupport::TestCase
       new_command.remove_images.join(" ")
   end
 
-  test "tag_current_as_latest" do
+  test "tag_latest_image" do
     assert_equal \
       "docker tag dhh/app:999 dhh/app:latest",
-      new_command.tag_current_as_latest.join(" ")
+      new_command.tag_latest_image.join(" ")
+  end
+
+  test "tag_latest_image with destination" do
+    @destination = "staging"
+    assert_equal \
+      "docker tag dhh/app:999 dhh/app:latest-staging",
+      new_command.tag_latest_image.join(" ")
+  end
+
+  test "make_env_directory" do
+    assert_equal "mkdir -p .kamal/env/roles", new_command.make_env_directory.join(" ")
+  end
+
+  test "remove_env_file" do
+    assert_equal "rm -f .kamal/env/roles/app-web.env", new_command.remove_env_file.join(" ")
+  end
+
+  test "cord" do
+    assert_equal "docker inspect -f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}' app-web-123 | awk '$2 == \"/tmp/kamal-cord\" {print $1}'", new_command.cord(version: 123).join(" ")
+  end
+
+  test "tie cord" do
+    assert_equal "mkdir -p . ; touch cordfile", new_command.tie_cord("cordfile").join(" ")
+    assert_equal "mkdir -p corddir ; touch corddir/cordfile", new_command.tie_cord("corddir/cordfile").join(" ")
+    assert_equal "mkdir -p /corddir ; touch /corddir/cordfile", new_command.tie_cord("/corddir/cordfile").join(" ")
+  end
+
+  test "cut cord" do
+    assert_equal "rm -r corddir", new_command.cut_cord("corddir").join(" ")
+  end
+
+  test "extract assets" do
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/extracted/app-web-999", "&&",
+      :docker, :stop, "-t 1", "app-web-assets", "2> /dev/null", "|| true", "&&",
+      :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "dhh/app:999", "sleep 1000000", "&&",
+      :docker, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/assets/extracted/app-web-999", "&&",
+      :docker, :stop, "-t 1", "app-web-assets"
+    ], new_command(asset_path: "/public/assets").extract_assets
+  end
+
+  test "sync asset volumes" do
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-999", ".kamal/assets/volumes/app-web-999"
+    ], new_command(asset_path: "/public/assets").sync_asset_volumes
+
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-999", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-999", ".kamal/assets/volumes/app-web-998", "|| true", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-998", ".kamal/assets/volumes/app-web-999", "|| true"
+    ], new_command(asset_path: "/public/assets").sync_asset_volumes(old_version: 998)
+  end
+
+  test "clean up assets" do
+    assert_equal [
+      :find, ".kamal/assets/extracted", "-maxdepth 1", "-name", "'app-web-*'", "!", "-name", "app-web-999", "-exec rm -rf \"{}\" +", ";",
+      :find, ".kamal/assets/volumes", "-maxdepth 1", "-name", "'app-web-*'", "!", "-name", "app-web-999", "-exec rm -rf \"{}\" +"
+    ], new_command(asset_path: "/public/assets").clean_up_assets
   end
 
   private
-    def new_command(role: "web")
+    def new_command(role: "web", **additional_config)
-      Kamal::Commands::App.new(Kamal::Configuration.new(@config, destination: @destination, version: "999"), role: role)
+      config = Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999")
+      Kamal::Commands::App.new(config, role: config.role(role))
     end
 end

test/commands/auditor_test.rb

@@ -21,7 +21,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
       :echo,
       "[#{@recorded_at}] [#{@performer}]",
       "app removed container",
-      ">>", "kamal-app-audit.log"
+      ">>", ".kamal/app-audit.log"
     ], @auditor.record("app removed container")
   end
 
@@ -31,7 +31,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
         :echo,
         "[#{@recorded_at}] [#{@performer}] [staging]",
         "app removed container",
-        ">>", "kamal-app-staging-audit.log"
+        ">>", ".kamal/app-staging-audit.log"
       ], auditor.record("app removed container")
     end
   end
@@ -42,7 +42,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
         :echo,
         "[#{@recorded_at}] [#{@performer}] [web]",
         "app removed container",
-        ">>", "kamal-app-audit.log"
+        ">>", ".kamal/app-audit.log"
       ], auditor.record("app removed container")
     end
   end
@@ -52,7 +52,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
       :echo,
       "[#{@recorded_at}] [#{@performer}] [value]",
       "app removed container",
-      ">>", "kamal-app-audit.log"
+      ">>", ".kamal/app-audit.log"
     ], @auditor.record("app removed container", detail: "value")
   end
 

test/commands/builder_test.rb

@@ -6,10 +6,10 @@ class CommandsBuilderTest < ActiveSupport::TestCase
   end
 
   test "target multiarch by default" do
-    builder = new_builder_command(builder: { "cache" => { "type" => "gha" }})
+    builder = new_builder_command(builder: { "cache" => { "type" => "gha" } })
     assert_equal "multiarch", builder.name
     assert_equal \
-      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
       builder.push.join(" ")
   end
 
@@ -17,23 +17,31 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "multiarch" => false })
     assert_equal "native", builder.name
     assert_equal \
-      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
+      "git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
       builder.push.join(" ")
   end
 
   test "target native cached when multiarch is off and cache is set" do
-    builder = new_builder_command(builder: { "multiarch" => false, "cache" => { "type" => "gha" }})
+    builder = new_builder_command(builder: { "multiarch" => false, "cache" => { "type" => "gha" } })
     assert_equal "native/cached", builder.name
     assert_equal \
-      "docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "git archive --format=tar HEAD | docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
       builder.push.join(" ")
   end
 
   test "target multiarch remote when local and remote is set" do
-    builder = new_builder_command(builder: { "local" => { }, "remote" => { }, "cache" => { "type" => "gha" } })
+    builder = new_builder_command(builder: { "local" => {}, "remote" => {}, "cache" => { "type" => "gha" } })
     assert_equal "multiarch/remote", builder.name
     assert_equal \
-      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
+      builder.push.join(" ")
+  end
+
+  test "target multiarch local when arch is set" do
+    builder = new_builder_command(builder: { "local" => { "arch" => "amd64" } })
+    assert_equal "multiarch", builder.name
+    assert_equal \
+      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile -",
       builder.push.join(" ")
   end
 
@@ -41,7 +49,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "remote" => { "arch" => "amd64" }, "cache" => { "type" => "gha" } })
     assert_equal "native/remote", builder.name
     assert_equal \
-      "docker buildx build --push --platform linux/amd64 --builder kamal-app-native-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64 --builder kamal-app-native-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
       builder.push.join(" ")
   end
 
@@ -53,7 +61,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
   end
 
   test "build secrets" do
-    builder = new_builder_command(builder: { "secrets" => ["token_a", "token_b"] })
+    builder = new_builder_command(builder: { "secrets" => [ "token_a", "token_b" ] })
     assert_equal \
       "-t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"token_a\" --secret id=\"token_b\" --file Dockerfile",
       builder.target.build_options.join(" ")
@@ -85,21 +93,61 @@ class CommandsBuilderTest < ActiveSupport::TestCase
   test "native push with build args" do
     builder = new_builder_command(builder: { "multiarch" => false, "args" => { "a" => 1, "b" => 2 } })
     assert_equal \
-      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
+      "git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
       builder.push.join(" ")
   end
 
   test "multiarch push with build args" do
     builder = new_builder_command(builder: { "args" => { "a" => 1, "b" => 2 } })
     assert_equal \
-      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile .",
+      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile -",
       builder.push.join(" ")
   end
 
   test "native push with build secrets" do
     builder = new_builder_command(builder: { "multiarch" => false, "secrets" => [ "a", "b" ] })
     assert_equal \
-      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
+      "git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
+      builder.push.join(" ")
+  end
+
+  test "build with ssh agent socket" do
+    builder = new_builder_command(builder: { "ssh" => "default=$SSH_AUTH_SOCK" })
+
+    assert_equal \
+      "-t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --ssh default=$SSH_AUTH_SOCK",
+      builder.target.build_options.join(" ")
+  end
+
+  test "validate image" do
+    assert_equal "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:123 | grep -x app || (echo \"Image dhh/app:123 is missing the 'service' label\" && exit 1)", new_builder_command.validate_image.join(" ")
+  end
+
+  test "multiarch context build" do
+    builder = new_builder_command(builder: { "context" => "./foo" })
+    assert_equal \
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo",
+      builder.push.join(" ")
+  end
+
+  test "native context build" do
+    builder = new_builder_command(builder: { "multiarch" => false, "context" => "./foo" })
+    assert_equal \
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo && docker push dhh/app:123 && docker push dhh/app:latest",
+      builder.push.join(" ")
+  end
+
+  test "cached context build" do
+    builder = new_builder_command(builder: { "multiarch" => false, "context" => "./foo", "cache" => { "type" => "gha" } })
+    assert_equal \
+      "docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile ./foo",
+      builder.push.join(" ")
+  end
+
+  test "remote context build" do
+    builder = new_builder_command(builder: { "remote" => { "arch" => "amd64" }, "context" => "./foo" })
+    assert_equal \
+      "docker buildx build --push --platform linux/amd64 --builder kamal-app-native-remote -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo",
       builder.push.join(" ")
   end
 

test/commands/docker_test.rb

@@ -9,7 +9,7 @@ class CommandsDockerTest < ActiveSupport::TestCase
   end
 
   test "install" do
-    assert_equal "curl -fsSL https://get.docker.com | sh", @docker.install.join(" ")
+    assert_equal "sh -c 'curl -fsSL https://get.docker.com || wget -O - https://get.docker.com || echo \"exit 1\"' | sh", @docker.install.join(" ")
   end
 
   test "installed?" do
@@ -21,6 +21,6 @@ class CommandsDockerTest < ActiveSupport::TestCase
   end
 
   test "superuser?" do
-    assert_equal '[ "${EUID:-$(id -u)}" -eq 0 ]', @docker.superuser?.join(" ")
+    assert_equal '[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', @docker.superuser?.join(" ")
   end
 end

test/commands/healthcheck_test.rb

@@ -10,7 +10,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
       new_command.run.join(" ")
   end
 
@@ -18,7 +18,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "port" => 3001 }
 
     assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3001 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"curl -f http://localhost:3001/up || exit 1\" --health-interval \"1s\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 3999:3001 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3001/up || exit 1\" --health-interval \"1s\" dhh/app:123",
       new_command.run.join(" ")
   end
 
@@ -26,7 +26,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
     @destination = "staging"
 
     assert_equal \
-      "docker run --detach --name healthcheck-app-staging-123 --publish 3999:3000 --label service=healthcheck-app-staging -e KAMAL_CONTAINER_NAME=\"healthcheck-app-staging\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-staging-123 --publish 3999:3000 --label service=healthcheck-app-staging -e KAMAL_CONTAINER_NAME=\"healthcheck-app-staging\" --env-file .kamal/env/roles/app-web-staging.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
       new_command.run.join(" ")
   end
 
@@ -34,14 +34,15 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "cmd" => "/bin/up" }
 
     assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"/bin/up\" --health-interval \"1s\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/up\" --health-interval \"1s\" dhh/app:123",
       new_command.run.join(" ")
   end
 
   test "run with custom options" do
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere" } } }
+    @config[:healthcheck] = { "exposed_port" => 4999 }
     assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 4999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
       new_command.run.join(" ")
   end
 
@@ -91,6 +92,13 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
       new_command.logs.join(" ")
   end
 
+  test "logs with custom lines number" do
+    @config[:healthcheck] = { "log_lines" => 150 }
+    assert_equal \
+      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker logs --tail 150 2>&1",
+      new_command.logs.join(" ")
+  end
+
   test "logs with destination" do
     @destination = "staging"
 

test/commands/lock_test.rb

@@ -10,24 +10,24 @@ class CommandsLockTest < ActiveSupport::TestCase
 
   test "status" do
     assert_equal \
-      "stat kamal_lock-app > /dev/null && cat kamal_lock-app/details | base64 -d",
+      "stat .kamal/locks/app-production > /dev/null && cat .kamal/locks/app-production/details | base64 -d",
       new_command.status.join(" ")
   end
 
   test "acquire" do
     assert_match \
-      /mkdir kamal_lock-app && echo ".*" > kamal_lock-app\/details/m,
+      %r{mkdir \.kamal/locks/app-production && echo ".*" > \.kamal/locks/app-production/details}m,
       new_command.acquire("Hello", "123").join(" ")
   end
 
   test "release" do
     assert_match \
-      "rm kamal_lock-app/details && rm -r kamal_lock-app",
+      "rm .kamal/locks/app-production/details && rm -r .kamal/locks/app-production",
       new_command.release.join(" ")
   end
 
   private
     def new_command
-      Kamal::Commands::Lock.new(Kamal::Configuration.new(@config, version: "123"))
+      Kamal::Commands::Lock.new(Kamal::Configuration.new(@config, version: "123", destination: "production"))
     end
 end

test/commands/prune_test.rb

@@ -10,7 +10,7 @@ class CommandsPruneTest < ActiveSupport::TestCase
 
   test "dangling images" do
     assert_equal \
-      "docker image prune --force --filter label=service=app --filter dangling=true",
+      "docker image prune --force --filter label=service=app",
       new_command.dangling_images.join(" ")
   end
 
@@ -20,10 +20,20 @@ class CommandsPruneTest < ActiveSupport::TestCase
       new_command.tagged_images.join(" ")
   end
 
-  test "containers" do
+  test "app containers" do
     assert_equal \
       "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done",
-      new_command.containers.join(" ")
+      new_command.app_containers(retain: 5).join(" ")
+
+    assert_equal \
+      "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +4 | while read container_id; do docker rm $container_id; done",
+      new_command.app_containers(retain: 3).join(" ")
+  end
+
+  test "healthcheck containers" do
+    assert_equal \
+      "docker container prune --force --filter label=service=healthcheck-app",
+      new_command.healthcheck_containers.join(" ")
   end
 
   private

test/commands/registry_test.rb

@@ -15,7 +15,7 @@ class CommandsRegistryTest < ActiveSupport::TestCase
 
   test "registry login" do
     assert_equal \
-      "docker login hub.docker.com -u dhh -p secret",
+      "docker login hub.docker.com -u \"dhh\" -p \"secret\"",
       @registry.login.join(" ")
   end
 
@@ -24,7 +24,18 @@ class CommandsRegistryTest < ActiveSupport::TestCase
     @config[:registry]["password"] = [ "KAMAL_REGISTRY_PASSWORD" ]
 
     assert_equal \
-      "docker login hub.docker.com -u dhh -p more-secret",
+      "docker login hub.docker.com -u \"dhh\" -p \"more-secret\"",
+      @registry.login.join(" ")
+  ensure
+    ENV.delete("KAMAL_REGISTRY_PASSWORD")
+  end
+
+  test "registry login escape password" do
+    ENV["KAMAL_REGISTRY_PASSWORD"] = "more-secret'\""
+    @config[:registry]["password"] = [ "KAMAL_REGISTRY_PASSWORD" ]
+
+    assert_equal \
+      "docker login hub.docker.com -u \"dhh\" -p \"more-secret'\\\"\"",
       @registry.login.join(" ")
   ensure
     ENV.delete("KAMAL_REGISTRY_PASSWORD")
@@ -35,7 +46,7 @@ class CommandsRegistryTest < ActiveSupport::TestCase
     @config[:registry]["username"] = [ "KAMAL_REGISTRY_USERNAME" ]
 
     assert_equal \
-      "docker login hub.docker.com -u also-secret -p secret",
+      "docker login hub.docker.com -u \"also-secret\" -p \"secret\"",
       @registry.login.join(" ")
   ensure
     ENV.delete("KAMAL_REGISTRY_USERNAME")

test/commands/server_test.rb

@@ -0,0 +1,23 @@
+require "test_helper"
+
+class CommandsServerTest < ActiveSupport::TestCase
+  setup do
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
+    }
+  end
+
+  test "ensure run directory" do
+    assert_equal "mkdir -p .kamal", new_command.ensure_run_directory.join(" ")
+  end
+
+  test "ensure non default run directory" do
+    assert_equal "mkdir -p /var/run/kamal", new_command(run_directory: "/var/run/kamal").ensure_run_directory.join(" ")
+  end
+
+  private
+    def new_command(extra_config = {})
+      Kamal::Commands::Server.new(Kamal::Configuration.new(@config.merge(extra_config)))
+    end
+end

test/commands/traefik_test.rb

@@ -18,72 +18,72 @@ class CommandsTraefikTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["host_port"] = "8080"
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["publish"] = false
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with ports configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
-    @config[:traefik]["options"] = {"publish" => %w[9000:9000 9001:9001]}
+    @config[:traefik]["options"] = { "publish" => %w[9000:9000 9001:9001] }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with volumes configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
-    @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json] }
+    @config[:traefik]["options"] = { "volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json] }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with several options configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
-    @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json], "publish" => %w[8080:8080], "memory" => "512m"}
+    @config[:traefik]["options"] = { "volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json], "publish" => %w[8080:8080], "memory" => "512m" }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with labels configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["labels"] = { "traefik.http.routers.dashboard.service" => "api@internal", "traefik.http.routers.dashboard.middlewares" => "auth" }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --label traefik.http.routers.dashboard.service=\"api@internal\" --label traefik.http.routers.dashboard.middlewares=\"auth\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --label traefik.http.routers.dashboard.service=\"api@internal\" --label traefik.http.routers.dashboard.middlewares=\"auth\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with env configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock -e EXAMPLE_API_KEY=\"456\" --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -91,7 +91,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
     @config.delete(:traefik)
 
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"",
       new_command.run.join(" ")
   end
 
@@ -99,7 +99,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -107,7 +107,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
     @config[:traefik]["args"]["log.level"] = "ERROR"
 
     assert_equal \
-    "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"ERROR\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+    "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"ERROR\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -138,7 +138,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
   test "traefik logs since 2h" do
     assert_equal \
       "docker logs traefik  --since 2h --timestamps 2>&1",
-      new_command.logs(since: '2h').join(" ")
+      new_command.logs(since: "2h").join(" ")
   end
 
   test "traefik logs last 10 lines" do
@@ -150,7 +150,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
   test "traefik logs with grep hello!" do
     assert_equal \
       "docker logs traefik --timestamps 2>&1 | grep 'hello!'",
-      new_command.logs(grep: 'hello!').join(" ")
+      new_command.logs(grep: "hello!").join(" ")
   end
 
   test "traefik remove container" do
@@ -167,14 +167,28 @@ class CommandsTraefikTest < ActiveSupport::TestCase
 
   test "traefik follow logs" do
     assert_equal \
-      "ssh -t root@1.1.1.1 'docker logs traefik --timestamps --tail 10 --follow 2>&1'",
+      "ssh -t root@1.1.1.1 -p 22 'docker logs traefik --timestamps --tail 10 --follow 2>&1'",
       new_command.follow_logs(host: @config[:servers].first)
   end
 
   test "traefik follow logs with grep hello!" do
     assert_equal \
-      "ssh -t root@1.1.1.1 'docker logs traefik --timestamps --tail 10 --follow 2>&1 | grep \"hello!\"'",
+      "ssh -t root@1.1.1.1 -p 22 'docker logs traefik --timestamps --tail 10 --follow 2>&1 | grep \"hello!\"'",
-      new_command.follow_logs(host: @config[:servers].first, grep: 'hello!')
+      new_command.follow_logs(host: @config[:servers].first, grep: "hello!")
+  end
+
+  test "secrets io" do
+    @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
+
+    assert_equal "EXAMPLE_API_KEY=456\n", new_command.env.secrets_io.string
+  end
+
+  test "make_env_directory" do
+    assert_equal "mkdir -p .kamal/env/traefik", new_command.make_env_directory.join(" ")
+  end
+
+  test "remove_env_file" do
+    assert_equal "rm -f .kamal/env/traefik/traefik.env", new_command.remove_env_file.join(" ")
   end
 
   private

test/configuration/accessory_test.rb

@@ -20,7 +20,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
             },
             "secret" => [
               "MYSQL_ROOT_PASSWORD"
-            ],
+            ]
           },
           "files" => [
             "config/mysql/my.cnf:/etc/mysql/my.cnf",
@@ -49,6 +49,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
           }
         },
         "monitoring" => {
+          "service" => "custom-monitoring",
           "image" => "monitoring:latest",
           "roles" => [ "web" ],
           "port" => "4321:4321",
@@ -72,6 +73,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
   test "service name" do
     assert_equal "app-mysql", @config.accessory(:mysql).service_name
     assert_equal "app-redis", @config.accessory(:redis).service_name
+    assert_equal "custom-monitoring", @config.accessory(:monitoring).service_name
   end
 
   test "port" do
@@ -80,9 +82,9 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
   end
 
   test "host" do
-    assert_equal ["1.1.1.5"], @config.accessory(:mysql).hosts
+    assert_equal [ "1.1.1.5" ], @config.accessory(:mysql).hosts
-    assert_equal ["1.1.1.6", "1.1.1.7"], @config.accessory(:redis).hosts
+    assert_equal [ "1.1.1.6", "1.1.1.7" ], @config.accessory(:redis).hosts
-    assert_equal ["1.1.1.1", "1.1.1.2"], @config.accessory(:monitoring).hosts
+    assert_equal [ "1.1.1.1", "1.1.1.2" ], @config.accessory(:monitoring).hosts
   end
 
   test "missing host" do
@@ -105,29 +107,40 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
     assert_equal "Specify one of `host`, `hosts` or `roles` for accessory `mysql`", exception.message
   end
 
-  test "label args" do
+  test "all hosts" do
-    assert_equal ["--label", "service=\"app-mysql\""], @config.accessory(:mysql).label_args
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4", "1.1.1.5", "1.1.1.6", "1.1.1.7" ], @config.all_hosts
-    assert_equal ["--label", "service=\"app-redis\"", "--label", "cache=\"true\""], @config.accessory(:redis).label_args
   end
 
-  test "env args with secret" do
+  test "label args" do
+    assert_equal [ "--label", "service=\"app-mysql\"" ], @config.accessory(:mysql).label_args
+    assert_equal [ "--label", "service=\"app-redis\"", "--label", "cache=\"true\"" ], @config.accessory(:redis).label_args
+  end
+
+  test "env args" do
+    assert_equal [ "--env-file", ".kamal/env/accessories/app-mysql.env", "--env", "MYSQL_ROOT_HOST=\"%\"" ], @config.accessory(:mysql).env_args
+    assert_equal [ "--env-file", ".kamal/env/accessories/app-redis.env", "--env", "SOMETHING=\"else\"" ], @config.accessory(:redis).env_args
+  end
+
+  test "env with secrets" do
     ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
 
-    @config.accessory(:mysql).env_args.tap do |env_args|
+    expected_secrets_file = <<~ENV
-      assert_equal ["-e", "MYSQL_ROOT_PASSWORD=\"secret123\"", "-e", "MYSQL_ROOT_HOST=\"%\""], Kamal::Utils.unredacted(env_args)
+      MYSQL_ROOT_PASSWORD=secret123
-      assert_equal ["-e", "MYSQL_ROOT_PASSWORD=[REDACTED]", "-e", "MYSQL_ROOT_HOST=\"%\""], Kamal::Utils.redacted(env_args)
+    ENV
-    end
+
+    assert_equal expected_secrets_file, @config.accessory(:mysql).env.secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/accessories/app-mysql.env", "--env", "MYSQL_ROOT_HOST=\"%\"" ], @config.accessory(:mysql).env_args
   ensure
     ENV["MYSQL_ROOT_PASSWORD"] = nil
   end
 
-  test "env args without secret" do
+  test "env secrets path" do
-    assert_equal ["-e", "SOMETHING=\"else\""], @config.accessory(:redis).env_args
+    assert_equal ".kamal/env/accessories/app-mysql.env", @config.accessory(:mysql).env.secrets_file
   end
 
   test "volume args" do
-    assert_equal ["--volume", "$PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf", "--volume", "$PWD/app-mysql/docker-entrypoint-initdb.d/structure.sql:/docker-entrypoint-initdb.d/structure.sql", "--volume", "$PWD/app-mysql/data:/var/lib/mysql"], @config.accessory(:mysql).volume_args
+    assert_equal [ "--volume", "$PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf", "--volume", "$PWD/app-mysql/docker-entrypoint-initdb.d/structure.sql:/docker-entrypoint-initdb.d/structure.sql", "--volume", "$PWD/app-mysql/data:/var/lib/mysql" ], @config.accessory(:mysql).volume_args
-    assert_equal ["--volume", "/var/lib/redis:/data"], @config.accessory(:redis).volume_args
+    assert_equal [ "--volume", "/var/lib/redis:/data" ], @config.accessory(:redis).volume_args
   end
 
   test "dynamic file expansion" do
@@ -138,11 +151,17 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
     assert_match "%", @config.accessory(:mysql).files.keys[2].read
   end
 
-  test "directories" do
+  test "directory with a relative path" do
-    assert_equal({"$PWD/app-mysql/data"=>"/var/lib/mysql"}, @config.accessory(:mysql).directories)
+    @deploy[:accessories]["mysql"]["directories"] = [ "data:/var/lib/mysql" ]
+    assert_equal({ "$PWD/app-mysql/data"=>"/var/lib/mysql" }, @config.accessory(:mysql).directories)
+  end
+
+  test "directory with an absolute path" do
+    @deploy[:accessories]["mysql"]["directories"] = [ "/var/data/mysql:/var/lib/mysql" ]
+    assert_equal({ "/var/data/mysql"=>"/var/lib/mysql" }, @config.accessory(:mysql).directories)
   end
 
   test "options" do
-    assert_equal ["--cpus", "\"4\"", "--memory", "\"2GB\""], @config.accessory(:redis).option_args
+    assert_equal [ "--cpus", "\"4\"", "--memory", "\"2GB\"" ], @config.accessory(:redis).option_args
   end
 end

test/configuration/builder_test.rb

@@ -124,9 +124,9 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase
   end
 
   test "setting secrets" do
-    @deploy_with_builder_option[:builder] = { "secrets" => ["GITHUB_TOKEN"] }
+    @deploy_with_builder_option[:builder] = { "secrets" => [ "GITHUB_TOKEN" ] }
 
-    assert_equal ["GITHUB_TOKEN"], @config_with_builder_option.builder.secrets
+    assert_equal [ "GITHUB_TOKEN" ], @config_with_builder_option.builder.secrets
   end
 
   test "dockerfile" do
@@ -140,7 +140,7 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase
   end
 
   test "context" do
-    assert_equal ".", @config.builder.context
+    assert_equal "-", @config.builder.context
   end
 
   test "setting context" do
@@ -148,4 +148,14 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase
 
     assert_equal "..", @config_with_builder_option.builder.context
   end
+
+  test "ssh" do
+    assert_nil @config.builder.ssh
+  end
+
+  test "setting ssh params" do
+    @deploy_with_builder_option[:builder] = { "ssh" => "default=$SSH_AUTH_SOCK" }
+
+    assert_equal "default=$SSH_AUTH_SOCK", @config_with_builder_option.builder.ssh
+  end
 end

test/configuration/env_test.rb

@@ -0,0 +1,74 @@
+require "test_helper"
+
+class ConfigurationEnvTest < ActiveSupport::TestCase
+  require "test_helper"
+
+  test "simple" do
+    assert_config \
+      config: { "foo" => "bar", "baz" => "haz" },
+      clear: { "foo" => "bar", "baz" => "haz" },
+      secrets: {}
+  end
+
+  test "clear" do
+    assert_config \
+      config: { "clear" => { "foo" => "bar", "baz" => "haz" } },
+      clear: { "foo" => "bar", "baz" => "haz" },
+      secrets: {}
+  end
+
+  test "secret" do
+    ENV["PASSWORD"] = "hello"
+    env = Kamal::Configuration::Env.from_config config: { "secret" => [ "PASSWORD" ] }
+
+    assert_config \
+      config: { "secret" => [ "PASSWORD" ] },
+      clear: {},
+      secrets: { "PASSWORD" => "hello" }
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "missing secret" do
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_raises(KeyError) { Kamal::Configuration::Env.from_config(config: { "secret" => [ "PASSWORD" ] }).secrets }
+  end
+
+  test "secret and clear" do
+    ENV["PASSWORD"] = "hello"
+    config = {
+      "secret" => [ "PASSWORD" ],
+      "clear" => {
+        "foo" => "bar",
+        "baz" => "haz"
+      }
+    }
+
+    assert_config \
+      config: config,
+      clear: { "foo" => "bar", "baz" => "haz" },
+      secrets: { "PASSWORD" => "hello" }
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "stringIO conversion" do
+    env = {
+      "foo" => "bar",
+      "baz" => "haz"
+    }
+
+    assert_equal "foo=bar\nbaz=haz\n", \
+      StringIO.new(Kamal::EnvFile.new(env)).read
+  end
+
+  private
+    def assert_config(config:, clear:, secrets:)
+      env = Kamal::Configuration::Env.from_config config: config
+      assert_equal clear, env.clear
+      assert_equal secrets, env.secrets
+    end
+end

test/configuration/role_test.rb

@@ -38,11 +38,11 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
   end
 
   test "label args" do
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"workers\"" ], @config_with_roles.role(:workers).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"workers\"", "--label", "destination" ], @config_with_roles.role(:workers).label_args
   end
 
   test "special label args for web" do
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "destination", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.routers.app-web.priority=\"2\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args
   end
 
   test "custom labels" do
@@ -53,7 +53,8 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
   test "custom labels via role specialization" do
     @deploy_with_roles[:labels] = { "my.custom.label" => "50" }
     @deploy_with_roles[:servers]["workers"]["labels"] = { "my.custom.label" => "70" }
-    assert_equal "70", @config_with_roles.role(:workers).labels["my.custom.label"]
+    config_with_roles = Kamal::Configuration.new(@deploy_with_roles)
+    assert_equal "70", config_with_roles.role(:workers).labels["my.custom.label"]
   end
 
   test "overwriting default traefik label" do
@@ -66,12 +67,27 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       c[:servers]["beta"] = { "traefik" => "true", "hosts" => [ "1.1.1.5" ] }
     })
 
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "traefik.http.services.app-beta.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "destination", "--label", "traefik.http.services.app-beta.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.routers.app-beta.priority=\"2\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args
   end
 
   test "env overwritten by role" do
-    assert_equal "redis://a/b", @config_with_roles.role(:workers).env["REDIS_URL"]
+    assert_equal "redis://a/b", @config_with_roles.role(:workers).env.clear["REDIS_URL"]
-    assert_equal ["-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], @config_with_roles.role(:workers).env_args
+
+    assert_equal "\n", @config_with_roles.role(:workers).env.secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
+  end
+
+  test "container name" do
+    ENV["VERSION"] = "12345"
+
+    assert_equal "app-workers-12345", @config_with_roles.role(:workers).container_name
+    assert_equal "app-web-12345", @config_with_roles.role(:web).container_name
+  ensure
+    ENV.delete("VERSION")
+  end
+
+  test "env args" do
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
   end
 
   test "env secret overwritten by role" do
@@ -94,13 +110,18 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       ]
     }
 
+    config_with_roles = Kamal::Configuration.new(@deploy_with_roles)
+
     ENV["REDIS_PASSWORD"] = "secret456"
     ENV["DB_PASSWORD"] = "secret&\"123"
 
-    @config_with_roles.role(:workers).env_args.tap do |env_args|
+    expected_secrets_file = <<~ENV
-      assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "DB_PASSWORD=\"secret&\\\"123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Kamal::Utils.unredacted(env_args)
+      REDIS_PASSWORD=secret456
-      assert_equal ["-e", "REDIS_PASSWORD=[REDACTED]", "-e", "DB_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Kamal::Utils.redacted(env_args)
+      DB_PASSWORD=secret&\"123
-    end
+    ENV
+
+    assert_equal expected_secrets_file, config_with_roles.role(:workers).env.secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], config_with_roles.role(:workers).env_args
   ensure
     ENV["REDIS_PASSWORD"] = nil
     ENV["DB_PASSWORD"] = nil
@@ -117,12 +138,16 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       ]
     }
 
+    config_with_roles = Kamal::Configuration.new(@deploy_with_roles)
+
     ENV["DB_PASSWORD"] = "secret123"
 
-    @config_with_roles.role(:workers).env_args.tap do |env_args|
+    expected_secrets_file = <<~ENV
-      assert_equal ["-e", "DB_PASSWORD=\"secret123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Kamal::Utils.unredacted(env_args)
+      DB_PASSWORD=secret123
-      assert_equal ["-e", "DB_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Kamal::Utils.redacted(env_args)
+    ENV
-    end
+
+    assert_equal expected_secrets_file, config_with_roles.role(:workers).env.secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], config_with_roles.role(:workers).env_args
   ensure
     ENV["DB_PASSWORD"] = nil
   end
@@ -139,11 +164,116 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
 
     ENV["REDIS_PASSWORD"] = "secret456"
 
-    @config_with_roles.role(:workers).env_args.tap do |env_args|
+    expected_secrets_file = <<~ENV
-      assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Kamal::Utils.unredacted(env_args)
+      REDIS_PASSWORD=secret456
-      assert_equal ["-e", "REDIS_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Kamal::Utils.redacted(env_args)
+    ENV
-    end
+
+    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env.secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
   ensure
     ENV["REDIS_PASSWORD"] = nil
   end
+
+  test "env overwritten by role with secrets" do
+    @deploy_with_roles[:env] = {
+      "clear" => {
+        "REDIS_URL" => "redis://a/b"
+      },
+      "secret" => [
+        "REDIS_PASSWORD"
+      ]
+    }
+
+    @deploy_with_roles[:servers]["workers"]["env"] = {
+      "clear" => {
+        "REDIS_URL" => "redis://c/d"
+      }
+    }
+
+    config_with_roles = Kamal::Configuration.new(@deploy_with_roles)
+
+    ENV["REDIS_PASSWORD"] = "secret456"
+
+    expected_secrets_file = <<~ENV
+      REDIS_PASSWORD=secret456
+    ENV
+
+    assert_equal expected_secrets_file, config_with_roles.role(:workers).env.secrets_io.string
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://c/d\"" ], config_with_roles.role(:workers).env_args
+  ensure
+    ENV["REDIS_PASSWORD"] = nil
+  end
+
+  test "env secrets_file" do
+    assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).env.secrets_file
+  end
+
+  test "uses cord" do
+    assert @config_with_roles.role(:web).uses_cord?
+    assert_not @config_with_roles.role(:workers).uses_cord?
+  end
+
+  test "cord host file" do
+    assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}/cord}, @config_with_roles.role(:web).cord_host_file
+  end
+
+  test "cord volume" do
+    assert_equal "/tmp/kamal-cord", @config_with_roles.role(:web).cord_volume.container_path
+    assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}}, @config_with_roles.role(:web).cord_volume.host_path
+    assert_equal "--volume", @config_with_roles.role(:web).cord_volume.docker_args[0]
+    assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}:/tmp/kamal-cord}, @config_with_roles.role(:web).cord_volume.docker_args[1]
+  end
+
+  test "cord container file" do
+    assert_equal "/tmp/kamal-cord/cord", @config_with_roles.role(:web).cord_container_file
+  end
+
+  test "asset path and volume args" do
+    ENV["VERSION"] = "12345"
+    assert_nil @config_with_roles.role(:web).asset_volume_args
+    assert_nil @config_with_roles.role(:workers).asset_volume_args
+    assert_nil @config_with_roles.role(:web).asset_path
+    assert_nil @config_with_roles.role(:workers).asset_path
+    assert_not @config_with_roles.role(:web).assets?
+    assert_not @config_with_roles.role(:workers).assets?
+
+    config_with_assets = Kamal::Configuration.new(@deploy_with_roles.dup.tap { |c|
+      c[:asset_path] = "foo"
+    })
+    assert_equal "foo", config_with_assets.role(:web).asset_path
+    assert_equal "foo", config_with_assets.role(:workers).asset_path
+    assert_equal [ "--volume", "$(pwd)/.kamal/assets/volumes/app-web-12345:foo" ], config_with_assets.role(:web).asset_volume_args
+    assert_nil config_with_assets.role(:workers).asset_volume_args
+    assert config_with_assets.role(:web).assets?
+    assert_not config_with_assets.role(:workers).assets?
+
+    config_with_assets = Kamal::Configuration.new(@deploy_with_roles.dup.tap { |c|
+      c[:servers]["web"] = { "hosts" => [ "1.1.1.1", "1.1.1.2" ], "asset_path" => "bar" }
+    })
+    assert_equal "bar", config_with_assets.role(:web).asset_path
+    assert_nil config_with_assets.role(:workers).asset_path
+    assert_equal [ "--volume", "$(pwd)/.kamal/assets/volumes/app-web-12345:bar" ], config_with_assets.role(:web).asset_volume_args
+    assert_nil config_with_assets.role(:workers).asset_volume_args
+    assert config_with_assets.role(:web).assets?
+    assert_not config_with_assets.role(:workers).assets?
+
+  ensure
+    ENV.delete("VERSION")
+  end
+
+  test "asset extracted path" do
+    ENV["VERSION"] = "12345"
+    assert_equal ".kamal/assets/extracted/app-web-12345", @config_with_roles.role(:web).asset_extracted_path
+    assert_equal ".kamal/assets/extracted/app-workers-12345", @config_with_roles.role(:workers).asset_extracted_path
+  ensure
+    ENV.delete("VERSION")
+  end
+
+  test "asset volume path" do
+    ENV["VERSION"] = "12345"
+    assert_equal ".kamal/assets/volumes/app-web-12345", @config_with_roles.role(:web).asset_volume_path
+    assert_equal ".kamal/assets/volumes/app-workers-12345", @config_with_roles.role(:workers).asset_volume_path
+  ensure
+    ENV.delete("VERSION")
+  end
 end

test/configuration/ssh_test.rb

@@ -7,7 +7,7 @@ class ConfigurationSshTest < ActiveSupport::TestCase
       registry: { "username" => "dhh", "password" => "secret" },
       env: { "REDIS_URL" => "redis://x/y" },
       servers: [ "1.1.1.1", "1.1.1.2" ],
-      volumes: ["/local/path:/container/path"]
+      volumes: [ "/local/path:/container/path" ]
     }
 
     @config = Kamal::Configuration.new(@deploy)
@@ -22,6 +22,9 @@ class ConfigurationSshTest < ActiveSupport::TestCase
 
     config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "log_level" => "debug" }) })
     assert_equal 0, config.ssh.options[:logger].level
+
+    config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "port" => 2222 }) })
+    assert_equal 2222, config.ssh.options[:port]
   end
 
   test "ssh options with proxy host" do

test/configuration/sshkit_test.rb

@@ -7,7 +7,7 @@ class ConfigurationSshkitTest < ActiveSupport::TestCase
       registry: { "username" => "dhh", "password" => "secret" },
       env: { "REDIS_URL" => "redis://x/y" },
       servers: [ "1.1.1.1", "1.1.1.2" ],
-      volumes: ["/local/path:/container/path"]
+      volumes: [ "/local/path:/container/path" ]
     }
 
     @config = Kamal::Configuration.new(@deploy)

test/configuration/volume_test.rb

@@ -0,0 +1,13 @@
+require "test_helper"
+
+class ConfigurationVolumeTest < ActiveSupport::TestCase
+  test "docker args absolute" do
+    volume = Kamal::Configuration::Volume.new(host_path: "/root/foo/bar", container_path: "/assets")
+    assert_equal [ "--volume", "/root/foo/bar:/assets" ], volume.docker_args
+  end
+
+  test "docker args relative" do
+    volume = Kamal::Configuration::Volume.new(host_path: "foo/bar", container_path: "/assets")
+    assert_equal [ "--volume", "$(pwd)/foo/bar:/assets" ], volume.docker_args
+  end
+end

test/configuration_test.rb

@@ -10,7 +10,7 @@ class ConfigurationTest < ActiveSupport::TestCase
       registry: { "username" => "dhh", "password" => "secret" },
       env: { "REDIS_URL" => "redis://x/y" },
       servers: [ "1.1.1.1", "1.1.1.2" ],
-      volumes: ["/local/path:/container/path"]
+      volumes: [ "/local/path:/container/path" ]
     }
 
     @config = Kamal::Configuration.new(@deploy)
@@ -42,6 +42,17 @@ class ConfigurationTest < ActiveSupport::TestCase
     end
   end
 
+  test "service name valid" do
+    assert Kamal::Configuration.new(@deploy.tap { _1[:service] = "hey-app1_primary" }).valid?
+    assert Kamal::Configuration.new(@deploy.tap { _1[:service] = "MyApp" }).valid?
+  end
+
+  test "service name invalid" do
+    assert_raise(ArgumentError) do
+      Kamal::Configuration.new @deploy.tap { _1[:service] = "app.com" }
+    end
+  end
+
   test "roles" do
     assert_equal %w[ web ], @config.roles.collect(&:name)
     assert_equal %w[ web workers ], @config_with_roles.roles.collect(&:name)
@@ -54,13 +65,13 @@ class ConfigurationTest < ActiveSupport::TestCase
   end
 
   test "all hosts" do
-    assert_equal [ "1.1.1.1", "1.1.1.2"], @config.all_hosts
+    assert_equal [ "1.1.1.1", "1.1.1.2" ], @config.all_hosts
     assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], @config_with_roles.all_hosts
   end
 
-  test "primary web host" do
+  test "primary host" do
-    assert_equal "1.1.1.1", @config.primary_web_host
+    assert_equal "1.1.1.1", @config.primary_host
-    assert_equal "1.1.1.1", @config_with_roles.primary_web_host
+    assert_equal "1.1.1.1", @config_with_roles.primary_host
   end
 
   test "traefik hosts" do
@@ -75,25 +86,35 @@ class ConfigurationTest < ActiveSupport::TestCase
   test "version no git repo" do
     ENV.delete("VERSION")
 
-    @config.expects(:system).with("git rev-parse").returns(nil)
+    Kamal::Git.expects(:used?).returns(nil)
-    error = assert_raises(RuntimeError) { @config.version}
+    error = assert_raises(RuntimeError) { @config.version }
     assert_match /no git repository found/, error.message
   end
 
   test "version from git committed" do
     ENV.delete("VERSION")
 
-    @config.expects(:`).with("git rev-parse HEAD").returns("git-version")
+    Kamal::Git.expects(:revision).returns("git-version")
-    Kamal::Utils.expects(:uncommitted_changes).returns("")
+    Kamal::Git.expects(:uncommitted_changes).returns("")
     assert_equal "git-version", @config.version
   end
 
   test "version from git uncommitted" do
     ENV.delete("VERSION")
 
-    @config.expects(:`).with("git rev-parse HEAD").returns("git-version")
+    Kamal::Git.expects(:revision).returns("git-version")
-    Kamal::Utils.expects(:uncommitted_changes).returns("M   file\n")
+    Kamal::Git.expects(:uncommitted_changes).returns("M   file\n")
-    assert_match /^git-version_uncommitted_[0-9a-f]{16}$/, @config.version
+    assert_equal "git-version", @config.version
+  end
+
+  test "version from uncommitted context" do
+    ENV.delete("VERSION")
+
+    config = Kamal::Configuration.new(@deploy.tap { |c| c[:builder] = { "context" => "." } })
+
+    Kamal::Git.expects(:revision).returns("git-version")
+    Kamal::Git.expects(:uncommitted_changes).returns("M   file\n")
+    assert_match /^git-version_uncommitted_[0-9a-f]{16}$/, config.version
   end
 
   test "version from env" do
@@ -124,50 +145,8 @@ class ConfigurationTest < ActiveSupport::TestCase
     assert_equal "app-missing", @config.service_with_version
   end
 
-  test "env args" do
+  test "healthcheck service" do
-    assert_equal [ "-e", "REDIS_URL=\"redis://x/y\"" ], @config.env_args
+    assert_equal "healthcheck-app", @config.healthcheck_service
-  end
-
-  test "env args with clear and secrets" do
-    ENV["PASSWORD"] = "secret123"
-
-    config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!({
-      env: { "clear" => { "PORT" => "3000" }, "secret" => [ "PASSWORD" ] }
-    }) })
-
-    assert_equal [ "-e", "PASSWORD=\"secret123\"", "-e", "PORT=\"3000\"" ], Kamal::Utils.unredacted(config.env_args)
-    assert_equal [ "-e", "PASSWORD=[REDACTED]", "-e", "PORT=\"3000\"" ], Kamal::Utils.redacted(config.env_args)
-  ensure
-    ENV["PASSWORD"] = nil
-  end
-
-  test "env args with only clear" do
-    config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!({
-      env: { "clear" => { "PORT" => "3000" } }
-    }) })
-
-    assert_equal [ "-e", "PORT=\"3000\"" ], config.env_args
-  end
-
-  test "env args with only secrets" do
-    ENV["PASSWORD"] = "secret123"
-
-    config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!({
-      env: { "secret" => [ "PASSWORD" ] }
-    }) })
-
-    assert_equal [ "-e", "PASSWORD=\"secret123\"" ], Kamal::Utils.unredacted(config.env_args)
-    assert_equal [ "-e", "PASSWORD=[REDACTED]" ], Kamal::Utils.redacted(config.env_args)
-  ensure
-    ENV["PASSWORD"] = nil
-  end
-
-  test "env args with missing secret" do
-    assert_raises(KeyError) do
-      config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!({
-        env: { "secret" => [ "PASSWORD" ] }
-      }) }).ensure_env_available
-    end
   end
 
   test "valid config" do
@@ -207,22 +186,32 @@ class ConfigurationTest < ActiveSupport::TestCase
     end
   end
 
+  test "allow_empty_roles" do
+    assert_silent do
+      Kamal::Configuration.new @deploy.merge(servers: { "web" => %w[ web ], "workers" => { "hosts" => %w[ ] } }, allow_empty_roles: true)
+    end
+
+    assert_raises(ArgumentError) do
+      Kamal::Configuration.new @deploy.merge(servers: { "web" => %w[], "workers" => { "hosts" => %w[] } }, allow_empty_roles: true)
+    end
+  end
+
   test "volume_args" do
-    assert_equal ["--volume", "/local/path:/container/path"], @config.volume_args
+    assert_equal [ "--volume", "/local/path:/container/path" ], @config.volume_args
   end
 
   test "logging args default" do
-    assert_equal ["--log-opt", "max-size=\"10m\""], @config.logging_args
+    assert_equal [ "--log-opt", "max-size=\"10m\"" ], @config.logging_args
   end
 
   test "logging args with configured options" do
     config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!(logging: { "options" => { "max-size" => "100m", "max-file" => 5 } }) })
-    assert_equal ["--log-opt", "max-size=\"100m\"", "--log-opt", "max-file=\"5\""], @config.logging_args
+    assert_equal [ "--log-opt", "max-size=\"100m\"", "--log-opt", "max-file=\"5\"" ], @config.logging_args
   end
 
   test "logging args with configured driver and options" do
     config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!(logging: { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => 5 } }) })
-    assert_equal ["--log-driver", "\"local\"", "--log-opt", "max-size=\"100m\"", "--log-opt", "max-file=\"5\""], @config.logging_args
+    assert_equal [ "--log-driver", "\"local\"", "--log-opt", "max-size=\"100m\"", "--log-opt", "max-file=\"5\"" ], @config.logging_args
   end
 
   test "erb evaluation of yml config" do
@@ -248,22 +237,33 @@ class ConfigurationTest < ActiveSupport::TestCase
     end
   end
 
+  test "destination required" do
+    dest_config_file = Pathname.new(File.expand_path("fixtures/deploy_for_required_dest.yml", __dir__))
+
+    assert_raises(ArgumentError) do
+      config = Kamal::Configuration.create_from config_file: dest_config_file
+    end
+
+    assert_nothing_raised do
+      config = Kamal::Configuration.create_from config_file: dest_config_file, destination: "world"
+    end
+  end
+
   test "to_h" do
     expected_config = \
-      { :roles=>["web"],
+      { roles: [ "web" ],
-        :hosts=>["1.1.1.1", "1.1.1.2"],
+        hosts: [ "1.1.1.1", "1.1.1.2" ],
-        :primary_host=>"1.1.1.1",
+        primary_host: "1.1.1.1",
-        :version=>"missing",
+        version: "missing",
-        :repository=>"dhh/app",
+        repository: "dhh/app",
-        :absolute_image=>"dhh/app:missing",
+        absolute_image: "dhh/app:missing",
-        :service_with_version=>"app-missing",
+        service_with_version: "app-missing",
-        :env_args=>["-e", "REDIS_URL=\"redis://x/y\""],
+        ssh_options: { user: "root", port: 22, log_level: :fatal, keepalive: true, keepalive_interval: 30 },
-        :ssh_options=>{ :user=>"root", :auth_methods=>["publickey"], log_level: :fatal, keepalive: true, keepalive_interval: 30 },
+        sshkit: {},
-        :sshkit=>{},
+        volume_args: [ "--volume", "/local/path:/container/path" ],
-        :volume_args=>["--volume", "/local/path:/container/path"],
+        builder: {},
-        :builder=>{},
+        logging: [ "--log-opt", "max-size=\"10m\"" ],
-        :logging=>["--log-opt", "max-size=\"10m\""],
+        healthcheck: { "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 } }
-        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7 }}
 
     assert_equal expected_config, @config.to_h
   end
@@ -283,4 +283,59 @@ class ConfigurationTest < ActiveSupport::TestCase
       Kamal::Configuration.new(@deploy.tap { |c| c.merge!(minimum_version: "10000.0.0") })
     end
   end
+
+  test "run directory" do
+    config = Kamal::Configuration.new(@deploy)
+    assert_equal ".kamal", config.run_directory
+
+    config = Kamal::Configuration.new(@deploy.merge!(run_directory: "/root/kamal"))
+    assert_equal "/root/kamal", config.run_directory
+  end
+
+  test "run directory as docker volume" do
+    config = Kamal::Configuration.new(@deploy)
+    assert_equal "$(pwd)/.kamal", config.run_directory_as_docker_volume
+
+    config = Kamal::Configuration.new(@deploy.merge!(run_directory: "/root/kamal"))
+    assert_equal "/root/kamal", config.run_directory_as_docker_volume
+  end
+
+  test "run id" do
+    SecureRandom.expects(:hex).with(16).returns("09876543211234567890098765432112")
+    assert_equal "09876543211234567890098765432112", @config.run_id
+  end
+
+  test "asset path" do
+    assert_nil @config.asset_path
+    assert_equal "foo", Kamal::Configuration.new(@deploy.merge!(asset_path: "foo")).asset_path
+  end
+
+  test "primary role" do
+    assert_equal "web", @config.primary_role.name
+
+    config = Kamal::Configuration.new(@deploy_with_roles.deep_merge({
+      servers: { "alternate_web" => { "hosts" => [ "1.1.1.4", "1.1.1.5" ] } },
+      primary_role: "alternate_web" }))
+
+
+    assert_equal "alternate_web", config.primary_role.name
+    assert_equal "1.1.1.4", config.primary_host
+    assert config.role(:alternate_web).primary?
+    assert config.role(:alternate_web).running_traefik?
+  end
+
+  test "primary role missing" do
+    error = assert_raises(ArgumentError) do
+      Kamal::Configuration.new(@deploy.merge(primary_role: "bar"))
+    end
+    assert_match /bar isn't defined/, error.message
+  end
+
+  test "retain_containers" do
+    assert_equal 5, @config.retain_containers
+    config = Kamal::Configuration.new(@deploy_with_roles.merge(retain_containers: 2))
+    assert_equal 2, config.retain_containers
+
+    assert_raises(ArgumentError) { Kamal::Configuration.new(@deploy_with_roles.merge(retain_containers: 0)) }
+  end
 end