Add kamal proxy update

For easy updates from Traefik to kamal-proxy, add `kamal proxy update`.

This stops and removes Traefik and kamal-proxy containers (just in case
it is run after an update). Then it starts kamal-proxy and calls
`kamal-proxy deploy` to route to any app containers that are already
running.

It can be run with a rolling option and calls the `pre-proxy-reboot` and
`post-proxy-reboot` hooks for each host.

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kamal (1.4.0)
+    kamal (1.5.2)
       activesupport (>= 7.0)
       base64 (~> 0.2)
       bcrypt_pbkdf (~> 1.0)
@@ -9,7 +9,7 @@ PATH
       dotenv (~> 2.8)
       ed25519 (~> 1.2)
       net-ssh (~> 7.0)
-      sshkit (~> 1.21)
+      sshkit (>= 1.22.2, < 2.0)
       thor (~> 1.2)
       zeitwerk (~> 2.5)
 
@@ -75,6 +75,8 @@ GEM
     mutex_m (0.2.0)
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
+    net-sftp (4.0.0)
+      net-ssh (>= 5.0.0, < 8.0.0)
     net-ssh (7.2.1)
     nokogiri (1.16.0-arm64-darwin)
       racc (~> 1.4)
@@ -151,9 +153,11 @@ GEM
       rubocop-rails
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
-    sshkit (1.21.7)
+    sshkit (1.22.2)
+      base64
       mutex_m
       net-scp (>= 1.1.2)
+      net-sftp (>= 2.1.2)
       net-ssh (>= 2.8.0)
     stringio (3.1.0)
     thor (1.3.0)

README.md

@@ -1,6 +1,6 @@
 # Kamal: Deploy web apps anywhere
 
-From bare metal to cloud VMs, deploy web apps anywhere with zero downtime. Kamal has the dynamic reverse-proxy Traefik hold requests while a new app container is started and the old one is stopped. Works seamlessly across multiple hosts, using SSHKit to execute commands. Originally built for Rails apps, Kamal will work with any type of web app that can be containerized with Docker.
+From bare metal to cloud VMs, deploy web apps anywhere with zero downtime. Kamal uses a [custom proxy](https://github.com/basecamp/kamal-proxy) for zero-downtime deployments. Works seamlessly across multiple hosts, using SSHKit to execute commands. Originally built for Rails apps, Kamal will work with any type of web app that can be containerized with Docker.
 
 ➡️ See [kamal-deploy.org](https://kamal-deploy.org) for documentation on [installation](https://kamal-deploy.org/docs/installation), [configuration](https://kamal-deploy.org/docs/configuration), and [commands](https://kamal-deploy.org/docs/commands).
 

kamal.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.executables = %w[ kamal ]
 
   spec.add_dependency "activesupport", ">= 7.0"
-  spec.add_dependency "sshkit", "~> 1.21"
+  spec.add_dependency "sshkit", ">= 1.22.2", "< 2.0"
   spec.add_dependency "net-ssh", "~> 7.0"
   spec.add_dependency "thor", "~> 1.2"
   spec.add_dependency "dotenv", "~> 2.8"

lib/kamal.rb

@@ -5,6 +5,6 @@ require "active_support"
 require "zeitwerk"
 
 loader = Zeitwerk::Loader.for_gem
-loader.ignore("#{__dir__}/kamal/sshkit_with_ext.rb")
+loader.ignore(File.join(__dir__, "kamal", "sshkit_with_ext.rb"))
 loader.setup
 loader.eager_load # We need all commands loaded.

lib/kamal/cli.rb

@@ -1,6 +1,7 @@
 module Kamal::Cli
   class LockError < StandardError; end
   class HookError < StandardError; end
+  class BootError < StandardError; end
 end
 
 # SSHKit uses instance eval, so we need a global const for ergonomics

lib/kamal/cli/accessory.rb

@@ -1,7 +1,7 @@
 class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "boot [NAME]", "Boot new accessory service on host (use NAME=all to boot all accessories)"
   def boot(name, login: true)
-    mutating do
+    with_lock do
       if name == "all"
         KAMAL.accessory_names.each { |accessory_name| boot(accessory_name) }
       else
@@ -21,7 +21,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "upload [NAME]", "Upload accessory files to host", hide: true
   def upload(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           accessory.files.each do |(local, remote)|
@@ -38,7 +38,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "directories [NAME]", "Create accessory directories on host", hide: true
   def directories(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           accessory.directories.keys.each do |host_path|
@@ -51,7 +51,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container; use NAME=all to boot all accessories)"
   def reboot(name)
-    mutating do
+    with_lock do
       if name == "all"
         KAMAL.accessory_names.each { |accessory_name| reboot(accessory_name) }
       else
@@ -70,7 +70,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "start [NAME]", "Start existing accessory container on host"
   def start(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *KAMAL.auditor.record("Started #{name} accessory"), verbosity: :debug
@@ -82,7 +82,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "stop [NAME]", "Stop existing accessory container on host"
   def stop(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *KAMAL.auditor.record("Stopped #{name} accessory"), verbosity: :debug
@@ -94,7 +94,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "restart [NAME]", "Restart existing accessory container on host"
   def restart(name)
-    mutating do
+    with_lock do
       with_accessory(name) do
         stop(name)
         start(name)
@@ -107,8 +107,9 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     if name == "all"
       KAMAL.accessory_names.each { |accessory_name| details(accessory_name) }
     else
+      type = "Accessory #{name}"
       with_accessory(name) do |accessory, hosts|
-        on(hosts) { puts capture_with_info(*accessory.info) }
+        on(hosts) { puts_by_host host, capture_with_info(*accessory.info), type: type }
       end
     end
   end
@@ -173,17 +174,12 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "remove [NAME]", "Remove accessory container, image and data directory from host (use NAME=all to remove all accessories)"
   option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def remove(name)
-    mutating do
-      if name == "all"
-        KAMAL.accessory_names.each { |accessory_name| remove(accessory_name) }
-      else
     confirming "This will remove all containers, images and data directories for #{name}. Are you sure?" do
-          with_accessory(name) do
+      with_lock do
-            stop(name)
+        if name == "all"
-            remove_container(name)
+          KAMAL.accessory_names.each { |accessory_name| remove_accessory(accessory_name) }
-            remove_image(name)
+        else
-            remove_service_directory(name)
+          remove_accessory(name)
-          end
         end
       end
     end
@@ -191,7 +187,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "remove_container [NAME]", "Remove accessory container from host", hide: true
   def remove_container(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *KAMAL.auditor.record("Remove #{name} accessory container"), verbosity: :debug
@@ -203,7 +199,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "remove_image [NAME]", "Remove accessory image from host", hide: true
   def remove_image(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *KAMAL.auditor.record("Removed #{name} accessory image"), verbosity: :debug
@@ -215,7 +211,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "remove_service_directory [NAME]", "Remove accessory directory used for uploaded files and data directories from host", hide: true
   def remove_service_directory(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *accessory.remove_service_directory
@@ -249,4 +245,13 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
         accessory.hosts
       end
     end
+
+    def remove_accessory(name)
+      with_accessory(name) do
+        stop(name)
+        remove_container(name)
+        remove_image(name)
+        remove_service_directory(name)
+      end
+    end
 end

lib/kamal/cli/app.rb

@@ -1,27 +1,31 @@
 class Kamal::Cli::App < Kamal::Cli::Base
   desc "boot", "Boot app on servers (or reboot app if already running)"
   def boot
-    mutating do
+    with_lock do
-      hold_lock_on_error do
       say "Get most recent version available as an image...", :magenta unless options[:version]
       using_version(version_or_latest) do |version|
         say "Start container with version #{version} using a #{KAMAL.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
 
         # Assets are prepared in a separate step to ensure they are on all hosts before booting
         on(KAMAL.hosts) do
-            execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
-            execute *KAMAL.app.tag_current_image_as_latest
-
           KAMAL.roles_on(host).each do |role|
-              Kamal::Cli::App::PrepareAssets.new(host, role, self).run
+            PrepareAssets.new(host, role, self).run
           end
         end
 
+        # Primary hosts and roles are returned first, so they can open the barrier
+        barrier = Barrier.new if KAMAL.roles.many?
+
         on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
           KAMAL.roles_on(host).each do |role|
-              Kamal::Cli::App::Boot.new(host, role, version, self).run
+            Boot.new(host, role, self, version, barrier).run
           end
         end
+
+        # Tag once the app booted on all hosts
+        on(KAMAL.hosts) do |host|
+          execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
+          execute *KAMAL.app.tag_latest_image
         end
       end
     end
@@ -29,13 +33,22 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "start", "Start existing app container on servers"
   def start
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
+          app = KAMAL.app(role: role, host: host)
           execute *KAMAL.auditor.record("Started app version #{KAMAL.config.version}"), verbosity: :debug
-          execute *KAMAL.app(role: role).start, raise_on_non_zero_exit: false
+          execute *app.start, raise_on_non_zero_exit: false
+
+          if role.running_proxy?
+            version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+            endpoint = capture_with_info(*app.container_endpoint(version: version)).strip
+            raise Kamal::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
+
+            execute *KAMAL.proxy.deploy(role.container_prefix, target: endpoint)
+          end
         end
       end
     end
@@ -43,13 +56,24 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "stop", "Stop app container on servers"
   def stop
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
+          app = KAMAL.app(role: role, host: host)
+          version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+
           execute *KAMAL.auditor.record("Stopped app", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).stop, raise_on_non_zero_exit: false
+
+          if role.running_proxy?
+            endpoint = capture_with_info(*app.container_endpoint(version: version)).strip
+            if endpoint.present?
+              execute *KAMAL.proxy.remove(role.container_prefix, target: endpoint), raise_on_non_zero_exit: false
+            end
+          end
+
+          execute *app.stop, raise_on_non_zero_exit: false
         end
       end
     end
@@ -62,21 +86,23 @@ class Kamal::Cli::App < Kamal::Cli::Base
       roles = KAMAL.roles_on(host)
 
       roles.each do |role|
-        puts_by_host host, capture_with_info(*KAMAL.app(role: role).info)
+        puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).info)
       end
     end
   end
 
-  desc "exec [CMD]", "Execute a custom command on servers (use --help to show options)"
+  desc "exec [CMD]", "Execute a custom command on servers within the app container (use --help to show options)"
   option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
   option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
+  option :env, aliases: "-e", type: :hash, desc: "Set environment variables for the command"
   def exec(cmd)
+    env = options[:env]
     case
     when options[:interactive] && options[:reuse]
       say "Get current version of running container...", :magenta unless options[:version]
       using_version(options[:version] || current_running_version) do |version|
         say "Launching interactive command with version #{version} via SSH from existing container on #{KAMAL.primary_host}...", :magenta
-        run_locally { exec KAMAL.app(role: KAMAL.primary_role).execute_in_existing_container_over_ssh(cmd, host: KAMAL.primary_host) }
+        run_locally { exec KAMAL.app(role: KAMAL.primary_role, host: KAMAL.primary_host).execute_in_existing_container_over_ssh(cmd, env: env) }
       end
 
     when options[:interactive]
@@ -84,7 +110,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
       using_version(version_or_latest) do |version|
         say "Launching interactive command with version #{version} via SSH from new container on #{KAMAL.primary_host}...", :magenta
         run_locally do
-          exec KAMAL.app(role: KAMAL.primary_role).execute_in_new_container_over_ssh(cmd, host: KAMAL.primary_host)
+          exec KAMAL.app(role: KAMAL.primary_role, host: KAMAL.primary_host).execute_in_new_container_over_ssh(cmd, env: env)
         end
       end
 
@@ -98,7 +124,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
           roles.each do |role|
             execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}", role: role), verbosity: :debug
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_existing_container(cmd))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).execute_in_existing_container(cmd, env: env))
           end
         end
       end
@@ -112,7 +138,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
           roles.each do |role|
             execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_new_container(cmd))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).execute_in_new_container(cmd, env: env))
           end
         end
       end
@@ -127,19 +153,21 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "stale_containers", "Detect app stale containers"
   option :stop, aliases: "-s", type: :boolean, default: false, desc: "Stop the stale containers found"
   def stale_containers
-    mutating do
     stop = options[:stop]
 
-      cli = self
+    with_lock_if_stopping do
-
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
-          cli.send(:stale_versions, host: host, role: role).each do |version|
+          app = KAMAL.app(role: role, host: host)
+          versions = capture_with_info(*app.list_versions, raise_on_non_zero_exit: false).split("\n")
+          versions -= [ capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip ]
+
+          versions.each do |version|
             if stop
               puts_by_host host, "Stopping stale container for role #{role} with version #{version}"
-              execute *KAMAL.app(role: role).stop(version: version), raise_on_non_zero_exit: false
+              execute *app.stop(version: version), raise_on_non_zero_exit: false
             else
               puts_by_host host,  "Detected stale container for role #{role} with version #{version} (use `kamal app stale_containers --stop` to stop)"
             end
@@ -173,8 +201,9 @@ class Kamal::Cli::App < Kamal::Cli::Base
         KAMAL.specific_roles ||= [ "web" ]
         role = KAMAL.roles_on(KAMAL.primary_host).first
 
-        info KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
+        app = KAMAL.app(role: role, host: host)
-        exec KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
+        info app.follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
+        exec app.follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
       end
     else
       lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
@@ -184,7 +213,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
         roles.each do |role|
           begin
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).logs(since: since, lines: lines, grep: grep))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).logs(since: since, lines: lines, grep: grep))
           rescue SSHKit::Command::Failed
             puts_by_host host, "Nothing found"
           end
@@ -195,7 +224,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove", "Remove app containers and images from servers"
   def remove
-    mutating do
+    with_lock do
       stop
       remove_containers
       remove_images
@@ -204,13 +233,13 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true
   def remove_container(version)
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
           execute *KAMAL.auditor.record("Removed app container with version #{version}", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).remove_container(version: version)
+          execute *KAMAL.app(role: role, host: host).remove_container(version: version)
         end
       end
     end
@@ -218,13 +247,13 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove_containers", "Remove all app containers from servers", hide: true
   def remove_containers
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
           execute *KAMAL.auditor.record("Removed all app containers", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).remove_containers
+          execute *KAMAL.app(role: role, host: host).remove_containers
         end
       end
     end
@@ -232,7 +261,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove_images", "Remove all app images from servers", hide: true
   def remove_images
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Removed all app images"), verbosity: :debug
         execute *KAMAL.app.remove_images
@@ -244,7 +273,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
   def version
     on(KAMAL.hosts) do |host|
       role = KAMAL.roles_on(host).first
-      puts_by_host host, capture_with_info(*KAMAL.app(role: role).current_running_version).strip
+      puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).current_running_version).strip
     end
   end
 
@@ -267,23 +296,20 @@ class Kamal::Cli::App < Kamal::Cli::Base
       version = nil
       on(host) do
         role = KAMAL.roles_on(host).first
-        version = capture_with_info(*KAMAL.app(role: role).current_running_version).strip
+        version = capture_with_info(*KAMAL.app(role: role, host: host).current_running_version).strip
       end
       version.presence
     end
 
-    def stale_versions(host:, role:)
+    def version_or_latest
-      versions = nil
+      options[:version] || KAMAL.config.latest_tag
-      on(host) do
-        versions = \
-          capture_with_info(*KAMAL.app(role: role).list_versions, raise_on_non_zero_exit: false)
-          .split("\n")
-          .drop(1)
-      end
-      versions
     end
 
-    def version_or_latest
+    def with_lock_if_stopping
-      options[:version] || "latest"
+      if options[:stop]
+        with_lock { yield }
+      else
+        yield
+      end
     end
 end

lib/kamal/cli/app/barrier.rb

@@ -0,0 +1,31 @@
+class Kamal::Cli::App::Barrier
+  def initialize
+    @ivar = Concurrent::IVar.new
+  end
+
+  def close
+    set(false)
+  end
+
+  def open
+    set(true)
+  end
+
+  def wait
+    unless opened?
+      raise Kamal::Cli::BootError.new("Halted at barrier")
+    end
+  end
+
+  private
+    def opened?
+      @ivar.value
+    end
+
+    def set(value)
+      @ivar.set(value)
+      true
+    rescue Concurrent::MultipleAssignmentError
+      false
+    end
+end

lib/kamal/cli/app/boot.rb

@@ -1,19 +1,30 @@
 class Kamal::Cli::App::Boot
-  attr_reader :host, :role, :version, :sshkit
+  attr_reader :host, :role, :version, :barrier, :sshkit
-  delegate :execute, :capture_with_info, :info, to: :sshkit
+  delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, to: :sshkit
-  delegate :uses_cord?, :assets?, to: :role
+  delegate :assets?, :running_proxy?, to: :role
 
-  def initialize(host, role, version, sshkit)
+  def initialize(host, role, sshkit, version, barrier)
     @host = host
     @role = role
     @version = version
+    @barrier = barrier
     @sshkit = sshkit
   end
 
   def run
     old_version = old_version_renamed_if_clashing
 
+    wait_at_barrier if queuer?
+
+    begin
       start_new_version
+    rescue => e
+      close_barrier if gatekeeper?
+      stop_new_version
+      raise
+    end
+
+    release_barrier if gatekeeper?
 
     if old_version
       stop_old_version(old_version)
@@ -21,18 +32,6 @@ class Kamal::Cli::App::Boot
   end
 
   private
-    def app
-      @app ||= KAMAL.app(role: role)
-    end
-
-    def auditor
-      @auditor = KAMAL.auditor(role: role)
-    end
-
-    def audit(message)
-      execute *auditor.record(message), verbosity: :debug
-    end
-
     def old_version_renamed_if_clashing
       if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
         renamed_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
@@ -46,22 +45,67 @@ class Kamal::Cli::App::Boot
 
     def start_new_version
       audit "Booted app version #{version}"
-      execute *app.tie_cord(role.cord_host_file) if uses_cord?
+      hostname = "#{host.to_s[0...51].gsub(/\.+$/, '')}-#{SecureRandom.hex(6)}"
-      execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
+      execute *app.run(hostname: hostname)
-      Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+      if running_proxy?
+        endpoint = capture_with_info(*app.container_endpoint(version: version)).strip
+        raise Kamal::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
+        execute *KAMAL.proxy.deploy(role.container_prefix, target: endpoint)
+      end
+    end
+
+    def stop_new_version
+      execute *app.stop(version: version), raise_on_non_zero_exit: false
     end
 
     def stop_old_version(version)
-      if uses_cord?
-        cord = capture_with_info(*app.cord(version: version), raise_on_non_zero_exit: false).strip
-        if cord.present?
-          execute *app.cut_cord(cord)
-          Kamal::Cli::Healthcheck::Poller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
-        end
-      end
-
       execute *app.stop(version: version), raise_on_non_zero_exit: false
-
       execute *app.clean_up_assets if assets?
     end
+
+    def release_barrier
+      if barrier.open
+        info "First #{KAMAL.primary_role} container is healthy on #{host}, booting other roles"
+      end
+    end
+
+    def wait_at_barrier
+      info "Waiting for the first healthy #{KAMAL.primary_role} container before booting #{role} on #{host}..."
+      barrier.wait
+      info "First #{KAMAL.primary_role} container is healthy, booting #{role} on #{host}..."
+    rescue Kamal::Cli::BootError
+      info "First #{KAMAL.primary_role} container is unhealthy, not booting #{role} on #{host}"
+      raise
+    end
+
+    def close_barrier
+      if barrier.close
+        info "First #{KAMAL.primary_role} container is unhealthy on #{host}, not booting other roles"
+        error capture_with_info(*app.logs(version: version))
+      end
+    end
+
+    def barrier_role?
+      role == KAMAL.primary_role
+    end
+
+    def app
+      @app ||= KAMAL.app(role: role, host: host)
+    end
+
+    def auditor
+      @auditor = KAMAL.auditor(role: role)
+    end
+
+    def audit(message)
+      execute *auditor.record(message), verbosity: :debug
+    end
+
+    def gatekeeper?
+      barrier && barrier_role?
+    end
+
+    def queuer?
+      barrier && !barrier_role?
+    end
 end

lib/kamal/cli/app/prepare_assets.rb

@@ -19,6 +19,6 @@ class Kamal::Cli::App::PrepareAssets
 
   private
     def app
-      @app ||= KAMAL.app(role: role)
+      @app ||= KAMAL.app(role: role, host: host)
     end
 end

lib/kamal/cli/base.rb

@@ -79,11 +79,10 @@ module Kamal::Cli
         puts "  Finished all in #{sprintf("%.1f seconds", runtime)}"
       end
 
-      def mutating
+      def with_lock
-        return yield if KAMAL.holding_lock?
+        if KAMAL.holding_lock?
-
+          yield
-        run_hook "pre-connect"
+        else
-
           ensure_run_and_locks_directory
 
           acquire_lock
@@ -91,17 +90,17 @@ module Kamal::Cli
           begin
             yield
           rescue
-          if KAMAL.hold_lock_on_error?
+            begin
-            error "  \e[31mDeploy lock was not released\e[0m"
-          else
               release_lock
+            rescue => e
+              say "Error releasing the deploy lock: #{e.message}", :red
             end
-
             raise
           end
 
           release_lock
         end
+      end
 
       def confirming(question)
         return yield if options[:confirmed]
@@ -141,29 +140,28 @@ module Kamal::Cli
         end
       end
 
-      def hold_lock_on_error
-        if KAMAL.hold_lock_on_error?
-          yield
-        else
-          KAMAL.hold_lock_on_error = true
-          yield
-          KAMAL.hold_lock_on_error = false
-        end
-      end
-
       def run_hook(hook, **extra_details)
         if !options[:skip_hooks] && KAMAL.hook.hook_exists?(hook)
           details = { hosts: KAMAL.hosts.join(","), command: command, subcommand: subcommand }
 
           say "Running the #{hook} hook...", :magenta
           run_locally do
-            KAMAL.with_verbosity(:debug) { execute *KAMAL.hook.run(hook, **details, **extra_details) }
+            execute *KAMAL.hook.run(hook, **details, **extra_details)
-          rescue SSHKit::Command::Failed
+          rescue SSHKit::Command::Failed => e
-            raise HookError.new("Hook `#{hook}` failed")
+            raise HookError.new("Hook `#{hook}` failed:\n#{e.message}")
           end
         end
       end
 
+      def on(*args, &block)
+        if !KAMAL.connected?
+          run_hook "pre-connect"
+          KAMAL.connected = true
+        end
+
+        super
+      end
+
       def command
         @kamal_command ||= begin
           invocation_class, invocation_commands = *first_invocation

lib/kamal/cli/build.rb

@@ -5,35 +5,47 @@ class Kamal::Cli::Build < Kamal::Cli::Base
 
   desc "deliver", "Build app and push app image to registry then pull image on servers"
   def deliver
-    mutating do
     push
     pull
   end
-  end
 
   desc "push", "Build and push app image to registry"
   def push
-    mutating do
     cli = self
 
     verify_local_dependencies
     run_hook "pre-build"
 
-      if (uncommitted_changes = Kamal::Git.uncommitted_changes).present?
+    uncommitted_changes = Kamal::Git.uncommitted_changes
-        say "The following paths have uncommitted changes:\n #{uncommitted_changes}", :yellow
+
+    if KAMAL.config.builder.git_clone?
+      if uncommitted_changes.present?
+        say "Building from a local git clone, so ignoring these uncommitted changes:\n #{uncommitted_changes}", :yellow
       end
 
+      run_locally do
+        Clone.new(self).prepare
+      end
+    elsif uncommitted_changes.present?
+      say "Building with uncommitted changes:\n #{uncommitted_changes}", :yellow
+    end
+
+    # Get the command here to ensure the Dir.chdir doesn't interfere with it
+    push = KAMAL.builder.push
+
     run_locally do
       begin
         KAMAL.with_verbosity(:debug) do
-            execute *KAMAL.builder.push
+          Dir.chdir(KAMAL.config.builder.build_directory) { execute *push }
         end
       rescue SSHKit::Command::Failed => e
         if e.message =~ /(no builder)|(no such file or directory)/
-            error "Missing compatible builder, so creating a new one first"
+          warn "Missing compatible builder, so creating a new one first"
 
           if cli.create
-              KAMAL.with_verbosity(:debug) { execute *KAMAL.builder.push }
+            KAMAL.with_verbosity(:debug) do
+              Dir.chdir(KAMAL.config.builder.build_directory) { execute *push }
+            end
           end
         else
           raise
@@ -41,11 +53,9 @@ class Kamal::Cli::Build < Kamal::Cli::Base
       end
     end
   end
-  end
 
   desc "pull", "Pull app image from registry onto servers"
   def pull
-    mutating do
     on(KAMAL.hosts) do
       execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
       execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
@@ -53,11 +63,9 @@ class Kamal::Cli::Build < Kamal::Cli::Base
       execute *KAMAL.builder.validate_image
     end
   end
-  end
 
   desc "create", "Create a build setup"
   def create
-    mutating do
     if (remote_host = KAMAL.config.builder.remote_host)
       connect_to_remote_host(remote_host)
     end
@@ -76,17 +84,14 @@ class Kamal::Cli::Build < Kamal::Cli::Base
       end
     end
   end
-  end
 
   desc "remove", "Remove build setup"
   def remove
-    mutating do
     run_locally do
       debug "Using builder: #{KAMAL.builder.name}"
       execute *KAMAL.builder.remove
     end
   end
-  end
 
   desc "details", "Show build setup"
   def details
@@ -114,8 +119,11 @@ class Kamal::Cli::Build < Kamal::Cli::Base
     def connect_to_remote_host(remote_host)
       remote_uri = URI.parse(remote_host)
       if remote_uri.scheme == "ssh"
-        options = { user: remote_uri.user, port: remote_uri.port }.compact
+        host = SSHKit::Host.new(
-        on(remote_uri.host, options) do
+          hostname: remote_uri.host,
+          ssh_options: { user: remote_uri.user, port: remote_uri.port }.compact
+        )
+        on(host, options) do
           execute "true"
         end
       end

lib/kamal/cli/build/clone.rb

@@ -0,0 +1,61 @@
+require "uri"
+
+class Kamal::Cli::Build::Clone
+  attr_reader :sshkit
+  delegate :info, :error, :execute, :capture_with_info, to: :sshkit
+
+  def initialize(sshkit)
+    @sshkit = sshkit
+  end
+
+  def prepare
+    begin
+      clone_repo
+    rescue SSHKit::Command::Failed => e
+      if e.message =~ /already exists and is not an empty directory/
+        reset
+      else
+        raise Kamal::Cli::Build::BuildError, "Failed to clone repo: #{e.message}"
+      end
+    end
+
+    validate!
+  rescue Kamal::Cli::Build::BuildError => e
+    error "Error preparing clone: #{e.message}, deleting and retrying..."
+
+    FileUtils.rm_rf KAMAL.config.builder.clone_directory
+    clone_repo
+    validate!
+  end
+
+  private
+    def clone_repo
+      info "Cloning repo into build directory `#{KAMAL.config.builder.build_directory}`..."
+
+      FileUtils.mkdir_p KAMAL.config.builder.clone_directory
+      execute *KAMAL.builder.clone
+    end
+
+    def reset
+      info "Resetting local clone as `#{KAMAL.config.builder.build_directory}` already exists..."
+
+      KAMAL.builder.clone_reset_steps.each { |step| execute *step }
+    rescue SSHKit::Command::Failed => e
+      raise Kamal::Cli::Build::BuildError, "Failed to clone repo: #{e.message}"
+    end
+
+    def validate!
+      status = capture_with_info(*KAMAL.builder.clone_status).strip
+
+      unless status.empty?
+        raise Kamal::Cli::Build::BuildError, "Clone in #{KAMAL.config.builder.build_directory} is dirty, #{status}"
+      end
+
+      revision = capture_with_info(*KAMAL.builder.clone_revision).strip
+      if revision != Kamal::Git.revision
+        raise Kamal::Cli::Build::BuildError, "Clone in #{KAMAL.config.builder.build_directory} is not on the correct revision, expected `#{Kamal::Git.revision}` but got `#{revision}`"
+      end
+    rescue SSHKit::Command::Failed => e
+      raise Kamal::Cli::Build::BuildError, "Failed to validate clone: #{e.message}"
+    end
+end

lib/kamal/cli/env.rb

@@ -3,21 +3,16 @@ require "tempfile"
 class Kamal::Cli::Env < Kamal::Cli::Base
   desc "push", "Push the env file to the remote hosts"
   def push
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pushed env files"), verbosity: :debug
 
         KAMAL.roles_on(host).each do |role|
-          execute *KAMAL.app(role: role).make_env_directory
+          execute *KAMAL.app(role: role, host: host).make_env_directory
-          upload! role.env.secrets_io, role.env.secrets_file, mode: 400
+          upload! role.env(host).secrets_io, role.env(host).secrets_file, mode: 400
         end
       end
 
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.traefik.make_env_directory
-        upload! KAMAL.traefik.env.secrets_io, KAMAL.traefik.env.secrets_file, mode: 400
-      end
-
       on(KAMAL.accessory_hosts) do
         KAMAL.accessories_on(host).each do |accessory|
           accessory_config = KAMAL.config.accessory(accessory)
@@ -30,19 +25,15 @@ class Kamal::Cli::Env < Kamal::Cli::Base
 
   desc "delete", "Delete the env file from the remote hosts"
   def delete
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Deleted env files"), verbosity: :debug
 
         KAMAL.roles_on(host).each do |role|
-          execute *KAMAL.app(role: role).remove_env_file
+          execute *KAMAL.app(role: role, host: host).remove_env_file
         end
       end
 
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.traefik.remove_env_file
-      end
-
       on(KAMAL.accessory_hosts) do
         KAMAL.accessories_on(host).each do |accessory|
           accessory_config = KAMAL.config.accessory(accessory)

lib/kamal/cli/healthcheck.rb

@@ -1,21 +0,0 @@
-class Kamal::Cli::Healthcheck < Kamal::Cli::Base
-  default_command :perform
-
-  desc "perform", "Health check current app version"
-  def perform
-    raise "The primary host is not configured to run Traefik" unless KAMAL.config.role(KAMAL.config.primary_role).running_traefik?
-    on(KAMAL.primary_host) do
-      begin
-        execute *KAMAL.healthcheck.run
-        Poller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
-      rescue Poller::HealthcheckError => e
-        error capture_with_info(*KAMAL.healthcheck.logs)
-        error capture_with_pretty_json(*KAMAL.healthcheck.container_health_log)
-        raise
-      ensure
-        execute *KAMAL.healthcheck.stop, raise_on_non_zero_exit: false
-        execute *KAMAL.healthcheck.remove, raise_on_non_zero_exit: false
-      end
-    end
-  end
-end

lib/kamal/cli/healthcheck/poller.rb

@@ -1,64 +0,0 @@
-module Kamal::Cli::Healthcheck::Poller
-  extend self
-
-  TRAEFIK_UPDATE_DELAY = 5
-
-  class HealthcheckError < StandardError; end
-
-  def wait_for_healthy(pause_after_ready: false, &block)
-    attempt = 1
-    max_attempts = KAMAL.config.healthcheck["max_attempts"]
-
-    begin
-      case status = block.call
-      when "healthy"
-        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
-      when "running" # No health check configured
-        sleep KAMAL.config.readiness_delay if pause_after_ready
-      else
-        raise HealthcheckError, "container not ready (#{status})"
-      end
-    rescue HealthcheckError => e
-      if attempt <= max_attempts
-        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-        sleep attempt
-        attempt += 1
-        retry
-      else
-        raise
-      end
-    end
-
-    info "Container is healthy!"
-  end
-
-  def wait_for_unhealthy(pause_after_ready: false, &block)
-    attempt = 1
-    max_attempts = KAMAL.config.healthcheck["max_attempts"]
-
-    begin
-      case status = block.call
-      when "unhealthy"
-        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
-      else
-        raise HealthcheckError, "container not unhealthy (#{status})"
-      end
-    rescue HealthcheckError => e
-      if attempt <= max_attempts
-        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-        sleep attempt
-        attempt += 1
-        retry
-      else
-        raise
-      end
-    end
-
-    info "Container is unhealthy!"
-  end
-
-  private
-    def info(message)
-      SSHKit.config.output.info(message)
-    end
-end

lib/kamal/cli/main.rb

@@ -3,14 +3,14 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def setup
     print_runtime do
-      mutating do
+      with_lock do
         invoke_options = deploy_options
 
         say "Ensure Docker is installed...", :magenta
         invoke "kamal:cli:server:bootstrap", [], invoke_options
 
-        say "Push env files...", :magenta
+        say "Evaluate and push env files...", :magenta
-        invoke "kamal:cli:env:push", [], invoke_options
+        invoke "kamal:cli:main:envify", [], invoke_options
 
         invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options
         deploy
@@ -22,7 +22,6 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def deploy
     runtime = print_runtime do
-      mutating do
       invoke_options = deploy_options
 
       say "Log into image registry...", :magenta
@@ -36,15 +35,11 @@ class Kamal::Cli::Main < Kamal::Cli::Base
         invoke "kamal:cli:build:deliver", [], invoke_options
       end
 
+      with_lock do
         run_hook "pre-deploy"
 
-        say "Ensure Traefik is running...", :magenta
+        say "Ensure proxy is running...", :magenta
-        invoke "kamal:cli:traefik:boot", [], invoke_options
+        invoke "kamal:cli:proxy:boot", [], invoke_options
-
-        if KAMAL.config.role(KAMAL.config.primary_role).running_traefik?
-          say "Ensure app can pass healthcheck...", :magenta
-          invoke "kamal:cli:healthcheck:perform", [], invoke_options
-        end
 
         say "Detect stale containers...", :magenta
         invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
@@ -59,11 +54,10 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     run_hook "post-deploy", runtime: runtime.round
   end
 
-  desc "redeploy", "Deploy app to servers without bootstrapping servers, starting Traefik, pruning, and registry login"
+  desc "redeploy", "Deploy app to servers without bootstrapping servers, starting proxy, pruning, and registry login"
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def redeploy
     runtime = print_runtime do
-      mutating do
       invoke_options = deploy_options
 
       if options[:skip_push]
@@ -74,11 +68,9 @@ class Kamal::Cli::Main < Kamal::Cli::Base
         invoke "kamal:cli:build:deliver", [], invoke_options
       end
 
+      with_lock do
         run_hook "pre-deploy"
 
-        say "Ensure app can pass healthcheck...", :magenta
-        invoke "kamal:cli:healthcheck:perform", [], invoke_options
-
         say "Detect stale containers...", :magenta
         invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
 
@@ -93,7 +85,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   def rollback(version)
     rolled_back = false
     runtime = print_runtime do
-      mutating do
+      with_lock do
         invoke_options = deploy_options
 
         KAMAL.config.version = version
@@ -115,7 +107,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
 
   desc "details", "Show details about all containers"
   def details
-    invoke "kamal:cli:traefik:details"
+    invoke "kamal:cli:proxy:details"
     invoke "kamal:cli:app:details"
     invoke "kamal:cli:accessory:details", [ "all" ]
   end
@@ -185,20 +177,24 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       env_path          = ".env"
     end
 
+    if Pathname.new(File.expand_path(env_template_path)).exist?
       File.write(env_path, ERB.new(File.read(env_template_path), trim_mode: "-").result, perm: 0600)
 
       unless options[:skip_push]
         reload_envs
         invoke "kamal:cli:env:push", options
       end
+    else
+      puts "Skipping envify (no #{env_template_path} exist)"
+    end
   end
 
-  desc "remove", "Remove Traefik, app, accessories, and registry session from servers"
+  desc "remove", "Remove proxy, app, accessories, and registry session from servers"
   option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def remove
-    mutating do
     confirming "This will remove all containers and images. Are you sure?" do
-        invoke "kamal:cli:traefik:remove", [], options.without(:confirmed)
+      with_lock do
+        invoke "kamal:cli:proxy:remove", [], options.without(:confirmed)
         invoke "kamal:cli:app:remove", [], options.without(:confirmed)
         invoke "kamal:cli:accessory:remove", [ "all" ], options
         invoke "kamal:cli:registry:logout", [], options.without(:confirmed)
@@ -223,9 +219,6 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   desc "env", "Manage environment files"
   subcommand "env", Kamal::Cli::Env
 
-  desc "healthcheck", "Healthcheck application"
-  subcommand "healthcheck", Kamal::Cli::Healthcheck
-
   desc "lock", "Manage the deploy lock"
   subcommand "lock", Kamal::Cli::Lock
 
@@ -238,19 +231,19 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   desc "server", "Bootstrap servers with curl and Docker"
   subcommand "server", Kamal::Cli::Server
 
-  desc "traefik", "Manage Traefik load balancer"
+  desc "proxy", "Manage load balancer proxy"
-  subcommand "traefik", Kamal::Cli::Traefik
+  subcommand "proxy", Kamal::Cli::Proxy
 
   private
     def container_available?(version)
       begin
         on(KAMAL.hosts) do
           KAMAL.roles_on(host).each do |role|
-            container_id = capture_with_info(*KAMAL.app(role: role).container_id_for_version(version))
+            container_id = capture_with_info(*KAMAL.app(role: role, host: host).container_id_for_version(version))
             raise "Container not found" unless container_id.present?
           end
         end
-      rescue SSHKit::Runner::ExecuteError => e
+      rescue SSHKit::Runner::ExecuteError, SSHKit::Runner::MultipleExecuteError => e
         if e.message =~ /Container not found/
           say "Error looking for container version #{version}: #{e.message}"
           return false

lib/kamal/cli/proxy.rb

@@ -0,0 +1,164 @@
+class Kamal::Cli::Proxy < Kamal::Cli::Base
+  desc "boot", "Boot proxy on servers"
+  def boot
+    with_lock do
+      on(KAMAL.proxy_hosts) do
+        execute *KAMAL.registry.login
+        execute *KAMAL.proxy.start_or_run
+      end
+    end
+  end
+
+  desc "reboot", "Reboot proxy on servers (stop container, remove container, start new container)"
+  option :rolling, type: :boolean, default: false, desc: "Reboot proxy on hosts in sequence, rather than in parallel"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  def reboot
+    confirming "This will cause a brief outage on each host. Are you sure?" do
+      with_lock do
+        host_groups = options[:rolling] ? KAMAL.proxy_hosts : [ KAMAL.proxy_hosts ]
+        host_groups.each do |hosts|
+          host_list = Array(hosts).join(",")
+          run_hook "pre-proxy-reboot", hosts: host_list
+          on(hosts) do
+            execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
+            execute *KAMAL.registry.login
+            execute *KAMAL.proxy.stop, raise_on_non_zero_exit: false
+            execute *KAMAL.proxy.remove_container
+            execute *KAMAL.proxy.run
+          end
+          run_hook "post-proxy-reboot", hosts: host_list
+        end
+      end
+    end
+  end
+
+  desc "start", "Start existing proxy container on servers"
+  def start
+    with_lock do
+      on(KAMAL.proxy_hosts) do
+        execute *KAMAL.auditor.record("Started proxy"), verbosity: :debug
+        execute *KAMAL.proxy.start
+      end
+    end
+  end
+
+  desc "stop", "Stop existing proxy container on servers"
+  def stop
+    with_lock do
+      on(KAMAL.proxy_hosts) do
+        execute *KAMAL.auditor.record("Stopped proxy"), verbosity: :debug
+        execute *KAMAL.proxy.stop, raise_on_non_zero_exit: false
+      end
+    end
+  end
+
+  desc "restart", "Restart existing proxy container on servers"
+  def restart
+    with_lock do
+      stop
+      start
+    end
+  end
+
+  desc "update", "Update from Traefik to kamal-proxy, for when moving from Kamal v1 to Kamal v2"
+  option :rolling, type: :boolean, default: false, desc: "Reboot proxy on hosts in sequence, rather than in parallel"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  def update
+    confirming "This will cause a brief outage on each host. Are you sure?" do
+      with_lock do
+        host_groups = options[:rolling] ? KAMAL.proxy_hosts : [ KAMAL.proxy_hosts ]
+        host_groups.each do |hosts|
+          host_list = Array(hosts).join(",")
+          run_hook "pre-proxy-reboot", hosts: host_list
+          on(hosts) do
+            info "Updating proxy from Traefik to kamal-proxy on #{host}..."
+            execute *KAMAL.auditor.record("Updated proxy from Traefik to kamal-proxy"), verbosity: :debug
+            execute *KAMAL.registry.login
+
+            info "Stopping and removing Traefik on #{host}..."
+            execute *KAMAL.proxy.stop(name: "traefik"), raise_on_non_zero_exit: false
+            execute *KAMAL.proxy.remove_container(filter: "label=org.opencontainers.image.title=traefik")
+            execute *KAMAL.proxy.remove_image(filter: "label=org.opencontainers.image.title=traefik")
+
+            info "Stopping and removing kamal-proxy on #{host}, if running..."
+            execute *KAMAL.proxy.stop, raise_on_non_zero_exit: false
+            execute *KAMAL.proxy.remove_container
+
+            info "Starting kamal-proxy on #{host}..."
+            execute *KAMAL.proxy.run
+
+            KAMAL.roles_on(host).select(&:running_proxy?).each do |role|
+              app = KAMAL.app(role: role, host: host)
+
+              version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+              endpoint = capture_with_info(*app.container_endpoint(version: version)).strip
+              raise Kamal::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, is the app container running?" if endpoint.empty?
+
+              info "Deploying #{endpoint} for role `#{role}` on #{host}..."
+              execute *KAMAL.proxy.deploy(role.container_prefix, target: endpoint)
+            end
+          end
+          run_hook "post-proxy-reboot", hosts: host_list
+        end
+      end
+    end
+  end
+
+  desc "details", "Show details about proxy container from servers"
+  def details
+    on(KAMAL.proxy_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.proxy.info), type: "Proxy" }
+  end
+
+  desc "logs", "Show log lines from proxy on servers"
+  option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
+  option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server"
+  option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
+  option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
+  def logs
+    grep = options[:grep]
+
+    if options[:follow]
+      run_locally do
+        info "Following logs on #{KAMAL.primary_host}..."
+        info KAMAL.proxy.follow_logs(host: KAMAL.primary_host, grep: grep)
+        exec KAMAL.proxy.follow_logs(host: KAMAL.primary_host, grep: grep)
+      end
+    else
+      since = options[:since]
+      lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
+
+      on(KAMAL.proxy_hosts) do |host|
+        puts_by_host host, capture(*KAMAL.proxy.logs(since: since, lines: lines, grep: grep)), type: "Proxy"
+      end
+    end
+  end
+
+  desc "remove", "Remove proxy container and image from servers"
+  def remove
+    with_lock do
+      stop
+      remove_container
+      remove_image
+    end
+  end
+
+  desc "remove_container", "Remove proxy container from servers", hide: true
+  def remove_container
+    with_lock do
+      on(KAMAL.proxy_hosts) do
+        execute *KAMAL.auditor.record("Removed proxy container"), verbosity: :debug
+        execute *KAMAL.proxy.remove_container
+      end
+    end
+  end
+
+  desc "remove_image", "Remove proxy image from servers", hide: true
+  def remove_image
+    with_lock do
+      on(KAMAL.proxy_hosts) do
+        execute *KAMAL.auditor.record("Removed proxy image"), verbosity: :debug
+        execute *KAMAL.proxy.remove_image
+      end
+    end
+  end
+end

lib/kamal/cli/prune.rb

@@ -1,7 +1,7 @@
 class Kamal::Cli::Prune < Kamal::Cli::Base
   desc "all", "Prune unused images and stopped containers"
   def all
-    mutating do
+    with_lock do
       containers
       images
     end
@@ -9,7 +9,7 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
 
   desc "images", "Prune unused images"
   def images
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pruned images"), verbosity: :debug
         execute *KAMAL.prune.dangling_images
@@ -24,11 +24,10 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
     retain = options.fetch(:retain, KAMAL.config.retain_containers)
     raise "retain must be at least 1" if retain < 1
 
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug
         execute *KAMAL.prune.app_containers(retain: retain)
-        execute *KAMAL.prune.healthcheck_containers
       end
     end
   end

lib/kamal/cli/server.rb

@@ -1,6 +1,29 @@
 class Kamal::Cli::Server < Kamal::Cli::Base
+  desc "exec", "Run a custom command on the server (use --help to show options)"
+  option :interactive, type: :boolean, aliases: "-i", default: false, desc: "Run the command interactively (use for console/bash)"
+  def exec(cmd)
+    hosts = KAMAL.hosts | KAMAL.accessory_hosts
+
+    case
+    when options[:interactive]
+      host = KAMAL.primary_host
+
+      say "Running '#{cmd}' on #{host} interactively...", :magenta
+
+      run_locally { exec KAMAL.server.run_over_ssh(cmd, host: host) }
+    else
+      say "Running '#{cmd}' on #{hosts.join(', ')}...", :magenta
+
+      on(hosts) do |host|
+        execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{host}"), verbosity: :debug
+        puts_by_host host, capture_with_info(cmd)
+      end
+    end
+  end
+
   desc "bootstrap", "Set up Docker to run Kamal apps"
   def bootstrap
+    with_lock do
       missing = []
 
       on(KAMAL.hosts | KAMAL.accessory_hosts) do |host|
@@ -23,3 +46,4 @@ class Kamal::Cli::Server < Kamal::Cli::Base
       run_hook "docker-setup"
     end
   end
+end

lib/kamal/cli/templates/deploy.yml

@@ -63,17 +63,6 @@ registry:
 #     directories:
 #       - data:/data
 
-# Configure custom arguments for Traefik. Be sure to reboot traefik when you modify it.
-# traefik:
-#   args:
-#     accesslog: true
-#     accesslog.format: json
-
-# Configure a custom healthcheck (default is /up on port 3000)
-# healthcheck:
-#   path: /healthz
-#   port: 4000
-
 # Bridge fingerprinted assets, like JS and CSS, between versions to avoid
 # hitting 404 on in-flight requests. Combines all files from new and old
 # version inside the asset_path.

lib/kamal/cli/templates/sample_hooks/post-proxy-reboot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooted proxy on $KAMAL_HOSTS"

lib/kamal/cli/templates/sample_hooks/post-traefik-reboot.sample

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-echo "Rebooted Traefik on $KAMAL_HOSTS"

lib/kamal/cli/templates/sample_hooks/pre-proxy-reboot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooting proxy on $KAMAL_HOSTS..."

lib/kamal/cli/templates/sample_hooks/pre-traefik-reboot.sample

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-echo "Rebooting Traefik on $KAMAL_HOSTS..."

lib/kamal/cli/traefik.rb

@@ -1,120 +0,0 @@
-class Kamal::Cli::Traefik < Kamal::Cli::Base
-  desc "boot", "Boot Traefik on servers"
-  def boot
-    mutating do
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.registry.login
-        execute *KAMAL.traefik.start_or_run
-      end
-    end
-  end
-
-  desc "reboot", "Reboot Traefik on servers (stop container, remove container, start new container)"
-  option :rolling, type: :boolean, default: false, desc: "Reboot traefik on hosts in sequence, rather than in parallel"
-  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
-  def reboot
-    confirming "This will cause a brief outage on each host. Are you sure?" do
-      mutating do
-        host_groups = options[:rolling] ? KAMAL.traefik_hosts : [ KAMAL.traefik_hosts ]
-        host_groups.each do |hosts|
-          host_list = Array(hosts).join(",")
-          run_hook "pre-traefik-reboot", hosts: host_list
-          on(hosts) do
-            execute *KAMAL.auditor.record("Rebooted traefik"), verbosity: :debug
-            execute *KAMAL.registry.login
-            execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
-            execute *KAMAL.traefik.remove_container
-            execute *KAMAL.traefik.run
-          end
-          run_hook "post-traefik-reboot", hosts: host_list
-        end
-      end
-    end
-  end
-
-  desc "start", "Start existing Traefik container on servers"
-  def start
-    mutating do
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.auditor.record("Started traefik"), verbosity: :debug
-        execute *KAMAL.traefik.start
-      end
-    end
-  end
-
-  desc "stop", "Stop existing Traefik container on servers"
-  def stop
-    mutating do
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.auditor.record("Stopped traefik"), verbosity: :debug
-        execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
-      end
-    end
-  end
-
-  desc "restart", "Restart existing Traefik container on servers"
-  def restart
-    mutating do
-      stop
-      start
-    end
-  end
-
-  desc "details", "Show details about Traefik container from servers"
-  def details
-    on(KAMAL.traefik_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.traefik.info), type: "Traefik" }
-  end
-
-  desc "logs", "Show log lines from Traefik on servers"
-  option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
-  option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server"
-  option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
-  option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
-  def logs
-    grep = options[:grep]
-
-    if options[:follow]
-      run_locally do
-        info "Following logs on #{KAMAL.primary_host}..."
-        info KAMAL.traefik.follow_logs(host: KAMAL.primary_host, grep: grep)
-        exec KAMAL.traefik.follow_logs(host: KAMAL.primary_host, grep: grep)
-      end
-    else
-      since = options[:since]
-      lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
-
-      on(KAMAL.traefik_hosts) do |host|
-        puts_by_host host, capture(*KAMAL.traefik.logs(since: since, lines: lines, grep: grep)), type: "Traefik"
-      end
-    end
-  end
-
-  desc "remove", "Remove Traefik container and image from servers"
-  def remove
-    mutating do
-      stop
-      remove_container
-      remove_image
-    end
-  end
-
-  desc "remove_container", "Remove Traefik container from servers", hide: true
-  def remove_container
-    mutating do
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.auditor.record("Removed traefik container"), verbosity: :debug
-        execute *KAMAL.traefik.remove_container
-      end
-    end
-  end
-
-  desc "remove_image", "Remove Traefik image from servers", hide: true
-  def remove_image
-    mutating do
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.auditor.record("Removed traefik image"), verbosity: :debug
-        execute *KAMAL.traefik.remove_image
-      end
-    end
-  end
-end

lib/kamal/commander.rb

@@ -2,12 +2,14 @@ require "active_support/core_ext/enumerable"
 require "active_support/core_ext/module/delegation"
 
 class Kamal::Commander
-  attr_accessor :verbosity, :holding_lock, :hold_lock_on_error
+  attr_accessor :verbosity, :holding_lock, :connected
+  delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :proxy_hosts, :accessory_hosts, to: :specifics
 
   def initialize
     self.verbosity = :info
     self.holding_lock = false
-    self.hold_lock_on_error = false
+    self.connected = false
+    @specifics = nil
   end
 
   def config
@@ -24,10 +26,12 @@ class Kamal::Commander
   attr_reader :specific_roles, :specific_hosts
 
   def specific_primary!
+    @specifics = nil
     self.specific_hosts = [ config.primary_host ]
   end
 
   def specific_roles=(role_names)
+    @specifics = nil
     if role_names.present?
       @specific_roles = Kamal::Utils.filter_specific_items(role_names, config.roles)
 
@@ -40,6 +44,7 @@ class Kamal::Commander
   end
 
   def specific_hosts=(hosts)
+    @specifics = nil
     if hosts.present?
       @specific_hosts = Kamal::Utils.filter_specific_items(hosts, config.all_hosts)
 
@@ -51,39 +56,6 @@ class Kamal::Commander
     end
   end
 
-  def primary_host
-    # Given a list of specific roles, make an effort to match up with the primary_role
-    specific_hosts&.first || specific_roles&.detect { |role| role == config.primary_role }&.primary_host || specific_roles&.first&.primary_host || config.primary_host
-  end
-
-  def primary_role
-    roles_on(primary_host).first
-  end
-
-  def roles
-    (specific_roles || config.roles).select do |role|
-      ((specific_hosts || config.all_hosts) & role.hosts).any?
-    end
-  end
-
-  def hosts
-    (specific_hosts || config.all_hosts).select do |host|
-      (specific_roles || config.roles).flat_map(&:hosts).include?(host)
-    end
-  end
-
-  def roles_on(host)
-    roles.select { |role| role.hosts.include?(host.to_s) }
-  end
-
-  def traefik_hosts
-    specific_hosts || config.traefik_hosts
-  end
-
-  def accessory_hosts
-    specific_hosts || config.accessories.flat_map(&:hosts)
-  end
-
   def accessory_names
     config.accessories&.collect(&:name) || []
   end
@@ -93,8 +65,8 @@ class Kamal::Commander
   end
 
 
-  def app(role: nil)
+  def app(role: nil, host: nil)
-    Kamal::Commands::App.new(config, role: role)
+    Kamal::Commands::App.new(config, role: role, host: host)
   end
 
   def accessory(name)
@@ -113,10 +85,6 @@ class Kamal::Commander
     @docker ||= Kamal::Commands::Docker.new(config)
   end
 
-  def healthcheck
-    @healthcheck ||= Kamal::Commands::Healthcheck.new(config)
-  end
-
   def hook
     @hook ||= Kamal::Commands::Hook.new(config)
   end
@@ -137,8 +105,8 @@ class Kamal::Commander
     @server ||= Kamal::Commands::Server.new(config)
   end
 
-  def traefik
+  def proxy
-    @traefik ||= Kamal::Commands::Traefik.new(config)
+    @proxy ||= Kamal::Commands::Proxy.new(config)
   end
 
 
@@ -166,8 +134,8 @@ class Kamal::Commander
     self.holding_lock
   end
 
-  def hold_lock_on_error?
+  def connected?
-    self.hold_lock_on_error
+    self.connected
   end
 
   private
@@ -181,4 +149,8 @@ class Kamal::Commander
       SSHKit.config.command_map[:docker] = "docker" # No need to use /usr/bin/env, just clogs up the logs
       SSHKit.config.output_verbosity = verbosity
     end
+
+    def specifics
+      @specifics ||= Kamal::Commander::Specifics.new(config, specific_hosts, specific_roles)
+    end
 end

lib/kamal/commander/specifics.rb

@@ -0,0 +1,49 @@
+class Kamal::Commander::Specifics
+  attr_reader :primary_host, :primary_role, :hosts, :roles
+  delegate :stable_sort!, to: Kamal::Utils
+
+  def initialize(config, specific_hosts, specific_roles)
+    @config, @specific_hosts, @specific_roles = config, specific_hosts, specific_roles
+
+    @roles, @hosts = specified_roles, specified_hosts
+
+    @primary_host = specific_hosts&.first || primary_specific_role&.primary_host || config.primary_host
+    @primary_role = primary_or_first_role(roles_on(primary_host))
+
+    stable_sort!(roles) { |role| role == primary_role ? 0 : 1 }
+    stable_sort!(hosts) { |host| roles_on(host).any? { |role| role == primary_role } ? 0 : 1 }
+  end
+
+  def roles_on(host)
+    roles.select { |role| role.hosts.include?(host.to_s) }
+  end
+
+  def proxy_hosts
+    config.proxy_hosts & specified_hosts
+  end
+
+  def accessory_hosts
+    specific_hosts || config.accessories.flat_map(&:hosts)
+  end
+
+  private
+    attr_reader :config, :specific_hosts, :specific_roles
+
+    def primary_specific_role
+      primary_or_first_role(specific_roles) if specific_roles.present?
+    end
+
+    def primary_or_first_role(roles)
+      roles.detect { |role| role == config.primary_role } || roles.first
+    end
+
+    def specified_roles
+      (specific_roles || config.roles) \
+        .select { |role| ((specific_hosts || config.all_hosts) & role.hosts).any? }
+    end
+
+    def specified_hosts
+      (specific_hosts || config.all_hosts) \
+        .select { |host| (specific_roles || config.roles).flat_map(&:hosts).include?(host) }
+    end
+end

lib/kamal/commands/app.rb

@@ -1,13 +1,14 @@
 class Kamal::Commands::App < Kamal::Commands::Base
-  include Assets, Containers, Cord, Execution, Images, Logging
+  include Assets, Containers, Execution, Images, Logging
 
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 
-  attr_reader :role, :role
+  attr_reader :role, :host
 
-  def initialize(config, role: nil)
+  def initialize(config, role: nil, host: nil)
     super(config)
     @role = role
+    @host = host
   end
 
   def run(hostname: nil)
@@ -18,8 +19,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
       *([ "--hostname", hostname ] if hostname),
       "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
       "-e", "KAMAL_VERSION=\"#{config.version}\"",
-      *role.env_args,
+      *role.env_args(host),
-      *role.health_check_args,
       *role.logging_args,
       *config.volume_args,
       *role.asset_volume_args,
@@ -49,30 +49,36 @@ class Kamal::Commands::App < Kamal::Commands::Base
 
 
   def current_running_container_id
-    docker :ps, "--quiet", *filter_args(statuses: ACTIVE_DOCKER_STATUSES), "--latest"
+    current_running_container(format: "--quiet")
   end
 
   def container_id_for_version(version, only_running: false)
     container_id_for(container_name: container_name(version), only_running: only_running)
   end
 
+  def container_name(version = nil)
+    [ role.container_prefix, version || config.version ].compact.join("-")
+  end
+
   def current_running_version
-    list_versions("--latest", statuses: ACTIVE_DOCKER_STATUSES)
+    pipe \
+      current_running_container(format: "--format '{{.Names}}'"),
+      extract_version_from_name
   end
 
   def list_versions(*docker_args, statuses: nil)
     pipe \
       docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
-      %(while read line; do echo ${line##{role.container_prefix}-}; done) # Extract SHA from "service-role-dest-SHA"
+      extract_version_from_name
   end
 
 
   def make_env_directory
-    make_directory role.env.secrets_directory
+    make_directory role.env(host).secrets_directory
   end
 
   def remove_env_file
-    [ :rm, "-f", role.env.secrets_file ]
+    [ :rm, "-f", role.env(host).secrets_file ]
   end
 
 
@@ -81,10 +87,33 @@ class Kamal::Commands::App < Kamal::Commands::Base
       [ role.container_prefix, version || config.version ].compact.join("-")
     end
 
+    def latest_image_id
+      docker :image, :ls, *argumentize("--filter", "reference=#{config.latest_image}"), "--format", "'{{.ID}}'"
+    end
+
+    def current_running_container(format:)
+      pipe \
+        shell(chain(latest_image_container(format: format), latest_container(format: format))),
+        [ :head, "-1" ]
+    end
+
+    def latest_image_container(format:)
+      latest_container format: format, filters: [ "ancestor=$(#{latest_image_id.join(" ")})" ]
+    end
+
+    def latest_container(format:, filters: nil)
+      docker :ps, "--latest", *format, *filter_args(statuses: ACTIVE_DOCKER_STATUSES), argumentize("--filter", filters)
+    end
+
     def filter_args(statuses: nil)
       argumentize "--filter", filters(statuses: statuses)
     end
 
+    def extract_version_from_name
+      # Extract SHA from "service-role-dest-SHA"
+      %(while read line; do echo ${line##{role.container_prefix}-}; done)
+    end
+
     def filters(statuses: nil)
       [ "label=service=#{config.service}" ].tap do |filters|
         filters << "label=destination=#{config.destination}" if config.destination

lib/kamal/commands/app/assets.rb

@@ -5,7 +5,7 @@ module Kamal::Commands::App::Assets
     combine \
       make_directory(role.asset_extracted_path),
       [ *docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true" ],
-      docker(:run, "--name", asset_container, "--detach", "--rm", config.latest_image, "sleep 1000000"),
+      docker(:run, "--name", asset_container, "--detach", "--rm", config.absolute_image, "sleep 1000000"),
       docker(:cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_path),
       docker(:stop, "-t 1", asset_container),
       by: "&&"

lib/kamal/commands/app/containers.rb

@@ -1,4 +1,6 @@
 module Kamal::Commands::App::Containers
+  DOCKER_HEALTH_LOG_FORMAT    = "'{{json .State.Health}}'"
+
   def list_containers
     docker :container, :ls, "--all", *filter_args
   end
@@ -20,4 +22,11 @@ module Kamal::Commands::App::Containers
   def remove_containers
     docker :container, :prune, "--force", *filter_args
   end
+
+  def container_endpoint(version:)
+    pipe \
+      container_id_for(container_name: container_name(version)),
+      xargs(docker(:inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'")),
+      [ :sed, "-e", "'s/\\/tcp$//'" ]
+  end
 end

lib/kamal/commands/app/cord.rb

@@ -1,22 +0,0 @@
-module Kamal::Commands::App::Cord
-  def cord(version:)
-    pipe \
-      docker(:inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", container_name(version)),
-      [ :awk, "'$2 == \"#{role.cord_volume.container_path}\" {print $1}'" ]
-  end
-
-  def tie_cord(cord)
-    create_empty_file(cord)
-  end
-
-  def cut_cord(cord)
-    remove_directory(cord)
-  end
-
-  private
-    def create_empty_file(file)
-      chain \
-        make_directory_for(file),
-        [ :touch, file ]
-    end
-end

lib/kamal/commands/app/execution.rb

@@ -1,27 +1,29 @@
 module Kamal::Commands::App::Execution
-  def execute_in_existing_container(*command, interactive: false)
+  def execute_in_existing_container(*command, interactive: false, env:)
     docker :exec,
       ("-it" if interactive),
+      *argumentize("--env", env),
       container_name,
       *command
   end
 
-  def execute_in_new_container(*command, interactive: false)
+  def execute_in_new_container(*command, interactive: false, env:)
     docker :run,
       ("-it" if interactive),
       "--rm",
-      *role&.env_args,
+      *role&.env_args(host),
+      *argumentize("--env", env),
       *config.volume_args,
       *role&.option_args,
       config.absolute_image,
       *command
   end
 
-  def execute_in_existing_container_over_ssh(*command, host:)
+  def execute_in_existing_container_over_ssh(*command,  env:)
-    run_over_ssh execute_in_existing_container(*command, interactive: true), host: host
+    run_over_ssh execute_in_existing_container(*command, interactive: true, env: env), host: host
   end
 
-  def execute_in_new_container_over_ssh(*command, host:)
+  def execute_in_new_container_over_ssh(*command, env:)
-    run_over_ssh execute_in_new_container(*command, interactive: true), host: host
+    run_over_ssh execute_in_new_container(*command, interactive: true, env: env), host: host
   end
 end

lib/kamal/commands/app/images.rb

@@ -7,7 +7,7 @@ module Kamal::Commands::App::Images
     docker :image, :prune, "--all", "--force", *filter_args
   end
 
-  def tag_current_image_as_latest
+  def tag_latest_image
     docker :tag, config.absolute_image, config.latest_image
   end
 end

lib/kamal/commands/app/logging.rb

@@ -1,7 +1,7 @@
 module Kamal::Commands::App::Logging
-  def logs(since: nil, lines: nil, grep: nil)
+  def logs(version: nil, since: nil, lines: nil, grep: nil)
     pipe \
-      current_running_container_id,
+      version ? container_id_for_version(version) : current_running_container_id,
       "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
       ("grep '#{grep}'" if grep)
   end

lib/kamal/commands/auditor.rb

@@ -21,7 +21,7 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
     def audit_log_file
       file = [ config.service, config.destination, "audit.log" ].compact.join("-")
 
-      "#{config.run_directory}/#{file}"
+      File.join(config.run_directory, file)
     end
 
     def audit_tags(**details)

lib/kamal/commands/base.rb

@@ -3,7 +3,6 @@ module Kamal::Commands
     delegate :sensitive, :argumentize, to: Kamal::Utils
 
     DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'"
-    DOCKER_HEALTH_LOG_FORMAT    = "'{{json .State.Health}}'"
 
     attr_accessor :config
 
@@ -18,7 +17,7 @@ module Kamal::Commands
         elsif config.ssh.proxy && config.ssh.proxy.is_a?(Net::SSH::Proxy::Command)
           cmd << " -o ProxyCommand='#{config.ssh.proxy.command_line_template}'"
         end
-        cmd << " -t #{config.ssh.user}@#{host} -p #{config.ssh.port} '#{command.join(" ")}'"
+        cmd << " -t #{config.ssh.user}@#{host} -p #{config.ssh.port} '#{command.join(" ").gsub("'", "'\\\\''")}'"
       end
     end
 
@@ -71,15 +70,15 @@ module Kamal::Commands
       end
 
       def shell(command)
-        [ :sh, "-c", "'#{command.flatten.join(" ").gsub("'", "'\\''")}'" ]
+        [ :sh, "-c", "'#{command.flatten.join(" ").gsub("'", "'\\\\''")}'" ]
       end
 
       def docker(*args)
         args.compact.unshift :docker
       end
 
-      def git(*args)
+      def git(*args, path: nil)
-        args.compact.unshift :git
+        [ :git, *([ "-C", path ] if path), *args.compact ]
       end
 
       def tags(**details)

lib/kamal/commands/builder.rb

@@ -3,6 +3,8 @@ require "active_support/core_ext/string/filters"
 class Kamal::Commands::Builder < Kamal::Commands::Base
   delegate :create, :remove, :push, :clean, :pull, :info, :validate_image, to: :target
 
+  include Clone
+
   def name
     target.class.to_s.remove("Kamal::Commands::Builder::").underscore.inquiry
   end

lib/kamal/commands/builder/base.rb

@@ -3,7 +3,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
   class BuilderError < StandardError; end
 
   delegate :argumentize, to: Kamal::Utils
-  delegate :args, :secrets, :dockerfile, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, :ssh, :git_archive?, to: :builder_config
+  delegate :args, :secrets, :dockerfile, :target, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, :ssh, to: :builder_config
 
   def clean
     docker :image, :rm, "--force", config.absolute_image
@@ -13,18 +13,8 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
     docker :pull, config.absolute_image
   end
 
-  def push
-    if git_archive?
-      pipe \
-        git(:archive, "--format=tar", :HEAD),
-        build_and_push
-    else
-      build_and_push
-    end
-  end
-
   def build_options
-    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_ssh ]
+    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_target, *build_ssh ]
   end
 
   def build_context
@@ -73,6 +63,10 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
       end
     end
 
+    def build_target
+      argumentize "--target", target if target.present?
+    end
+
     def build_ssh
       argumentize "--ssh", ssh if ssh.present?
     end

lib/kamal/commands/builder/clone.rb

@@ -0,0 +1,28 @@
+module Kamal::Commands::Builder::Clone
+  extend ActiveSupport::Concern
+
+  included do
+    delegate :clone_directory, :build_directory, to: :"config.builder"
+  end
+
+  def clone
+    git :clone, Kamal::Git.root, path: clone_directory
+  end
+
+  def clone_reset_steps
+    [
+      git(:remote, "set-url", :origin, Kamal::Git.root, path: build_directory),
+      git(:fetch, :origin, path: build_directory),
+      git(:reset, "--hard", Kamal::Git.revision, path: build_directory),
+      git(:clean, "-fdx", path: build_directory)
+    ]
+  end
+
+  def clone_status
+    git :status, "--porcelain", path: build_directory
+  end
+
+  def clone_revision
+    git :"rev-parse", :HEAD, path: build_directory
+  end
+end

lib/kamal/commands/builder/multiarch.rb

@@ -13,6 +13,15 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
       docker(:buildx, :ls)
   end
 
+  def push
+    docker :buildx, :build,
+      "--push",
+      "--platform", platform_names,
+      "--builder", builder_name,
+      *build_options,
+      build_context
+  end
+
   private
     def builder_name
       "kamal-#{config.service}-multiarch"
@@ -25,13 +34,4 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
         "linux/amd64,linux/arm64"
       end
     end
-
-    def build_and_push
-      docker :buildx, :build,
-        "--push",
-        "--platform", platform_names,
-        "--builder", builder_name,
-        *build_options,
-        build_context
-    end
 end

lib/kamal/commands/builder/native.rb

@@ -11,8 +11,7 @@ class Kamal::Commands::Builder::Native < Kamal::Commands::Builder::Base
     # No-op on native
   end
 
-  private
+  def push
-    def build_and_push
     combine \
       docker(:build, *build_options, build_context),
       docker(:push, config.absolute_image),

lib/kamal/commands/builder/native/cached.rb

@@ -7,8 +7,7 @@ class Kamal::Commands::Builder::Native::Cached < Kamal::Commands::Builder::Nativ
     docker :buildx, :rm, builder_name
   end
 
-  private
+  def push
-    def build_and_push
     docker :buildx, :build,
       "--push",
       *build_options,

lib/kamal/commands/builder/native/remote.rb

@@ -17,6 +17,15 @@ class Kamal::Commands::Builder::Native::Remote < Kamal::Commands::Builder::Nativ
       docker(:buildx, :ls)
   end
 
+  def push
+    docker :buildx, :build,
+    "--push",
+    "--platform", platform,
+    "--builder", builder_name,
+    *build_options,
+    build_context
+  end
+
 
   private
     def builder_name
@@ -47,13 +56,4 @@ class Kamal::Commands::Builder::Native::Remote < Kamal::Commands::Builder::Nativ
     def remove_buildx
       docker :buildx, :rm, builder_name
     end
-
-    def build_and_push
-      docker :buildx, :build,
-      "--push",
-      "--platform", platform,
-      "--builder", builder_name,
-      *build_options,
-      build_context
-    end
 end

lib/kamal/commands/healthcheck.rb

@@ -1,59 +0,0 @@
-class Kamal::Commands::Healthcheck < Kamal::Commands::Base
-  def run
-    primary = config.role(config.primary_role)
-
-    docker :run,
-      "--detach",
-      "--name", container_name_with_version,
-      "--publish", "#{exposed_port}:#{config.healthcheck["port"]}",
-      "--label", "service=#{config.healthcheck_service}",
-      "-e", "KAMAL_CONTAINER_NAME=\"#{config.healthcheck_service}\"",
-      *primary.env_args,
-      *primary.health_check_args(cord: false),
-      *config.volume_args,
-      *primary.option_args,
-      config.absolute_image,
-      primary.cmd
-  end
-
-  def status
-    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_STATUS_FORMAT))
-  end
-
-  def container_health_log
-    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT))
-  end
-
-  def logs
-    pipe container_id, xargs(docker(:logs, "--tail", log_lines, "2>&1"))
-  end
-
-  def stop
-    pipe container_id, xargs(docker(:stop))
-  end
-
-  def remove
-    pipe container_id, xargs(docker(:container, :rm))
-  end
-
-  private
-    def container_name_with_version
-      "#{config.healthcheck_service}-#{config.version}"
-    end
-
-    def container_id
-      container_id_for(container_name: container_name_with_version)
-    end
-
-    def health_url
-      "http://localhost:#{exposed_port}#{config.healthcheck["path"]}"
-    end
-
-    def exposed_port
-      config.healthcheck["exposed_port"]
-    end
-
-    def log_lines
-      config.healthcheck["log_lines"]
-    end
-end

lib/kamal/commands/hook.rb

@@ -9,6 +9,6 @@ class Kamal::Commands::Hook < Kamal::Commands::Base
 
   private
     def hook_file(hook)
-      "#{config.hooks_path}/#{hook}"
+      File.join(config.hooks_path, hook)
     end
 end

lib/kamal/commands/proxy.rb

@@ -0,0 +1,79 @@
+class Kamal::Commands::Proxy < Kamal::Commands::Base
+  delegate :argumentize, :optionize, to: Kamal::Utils
+  delegate :container_name, to: :proxy_config
+
+  attr_reader :proxy_config
+
+  def initialize(config)
+    super
+    @proxy_config = config.proxy
+  end
+
+  def run
+    docker :run,
+      "--name", container_name,
+      "--detach",
+      "--restart", "unless-stopped",
+      *proxy_config.publish_args,
+      "--volume", "/var/run/docker.sock:/var/run/docker.sock",
+      "--volume", "#{container_name}:/root/.config/kamal-proxy",
+      *config.logging_args,
+      *proxy_config.docker_options_args,
+      proxy_config.image
+  end
+
+  def start
+    docker :container, :start, container_name
+  end
+
+  def stop(name: container_name)
+    docker :container, :stop, name
+  end
+
+  def start_or_run
+    combine start, run, by: "||"
+  end
+
+  def deploy(service, target:)
+    optionize({ target: target })
+    docker :exec, container_name, "kamal-proxy", :deploy, service, *optionize({ target: target }), *proxy_config.deploy_command_args
+  end
+
+  def remove(service, target:)
+    docker :exec, container_name, "kamal-proxy", :remove, service, *optionize({ target: target })
+  end
+
+  def info
+    docker :ps, "--filter", "name=^#{container_name}$"
+  end
+
+  def logs(since: nil, lines: nil, grep: nil)
+    pipe \
+      docker(:logs, container_name, (" --since #{since}" if since), (" --tail #{lines}" if lines), "--timestamps", "2>&1"),
+      ("grep '#{grep}'" if grep)
+  end
+
+  def follow_logs(host:, grep: nil)
+    run_over_ssh pipe(
+      docker(:logs, container_name, "--timestamps", "--tail", "10", "--follow", "2>&1"),
+      (%(grep "#{grep}") if grep)
+    ).join(" "), host: host
+  end
+
+  def remove_container(filter: container_filter)
+    docker :container, :prune, "--force", "--filter", filter
+  end
+
+  def remove_image(filter: image_filter)
+    docker :image, :prune, "--all", "--force", "--filter", filter
+  end
+
+  private
+    def container_filter
+      "label=org.opencontainers.image.title=kamal-proxy"
+    end
+
+    def image_filter
+      "label=org.opencontainers.image.title=kamal-proxy"
+    end
+end

lib/kamal/commands/prune.rb

@@ -20,10 +20,6 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
       "while read container_id; do docker rm $container_id; done"
   end
 
-  def healthcheck_containers
-    docker :container, :prune, "--force", *healthcheck_service_filter
-  end
-
   private
     def stopped_containers_filters
       [ "created", "exited", "dead" ].flat_map { |status| [ "--filter", "status=#{status}" ] }
@@ -39,8 +35,4 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
     def service_filter
       [ "--filter", "label=service=#{config.service}" ]
     end
-
-    def healthcheck_service_filter
-      [ "--filter", "label=service=#{config.healthcheck_service}" ]
-    end
 end

lib/kamal/commands/traefik.rb

@@ -1,124 +0,0 @@
-class Kamal::Commands::Traefik < Kamal::Commands::Base
-  delegate :argumentize, :optionize, to: Kamal::Utils
-
-  DEFAULT_IMAGE = "traefik:v2.10"
-  CONTAINER_PORT = 80
-  DEFAULT_ARGS = {
-    "log.level" => "DEBUG"
-  }
-  DEFAULT_LABELS = {
-    # These ensure we serve a 502 rather than a 404 if no containers are available
-    "traefik.http.routers.catchall.entryPoints" => "http",
-    "traefik.http.routers.catchall.rule" => "PathPrefix(`/`)",
-    "traefik.http.routers.catchall.service" => "unavailable",
-    "traefik.http.routers.catchall.priority" => 1,
-    "traefik.http.services.unavailable.loadbalancer.server.port" => "0"
-  }
-
-  def run
-    docker :run, "--name traefik",
-      "--detach",
-      "--restart", "unless-stopped",
-      *publish_args,
-      "--volume", "/var/run/docker.sock:/var/run/docker.sock",
-      *env_args,
-      *config.logging_args,
-      *label_args,
-      *docker_options_args,
-      image,
-      "--providers.docker",
-      *cmd_option_args
-  end
-
-  def start
-    docker :container, :start, "traefik"
-  end
-
-  def stop
-    docker :container, :stop, "traefik"
-  end
-
-  def start_or_run
-    any start, run
-  end
-
-  def info
-    docker :ps, "--filter", "name=^traefik$"
-  end
-
-  def logs(since: nil, lines: nil, grep: nil)
-    pipe \
-      docker(:logs, "traefik", (" --since #{since}" if since), (" --tail #{lines}" if lines), "--timestamps", "2>&1"),
-      ("grep '#{grep}'" if grep)
-  end
-
-  def follow_logs(host:, grep: nil)
-    run_over_ssh pipe(
-      docker(:logs, "traefik", "--timestamps", "--tail", "10", "--follow", "2>&1"),
-      (%(grep "#{grep}") if grep)
-    ).join(" "), host: host
-  end
-
-  def remove_container
-    docker :container, :prune, "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
-  end
-
-  def remove_image
-    docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
-  end
-
-  def port
-    "#{host_port}:#{CONTAINER_PORT}"
-  end
-
-  def env
-    Kamal::Configuration::Env.from_config \
-      config: config.traefik.fetch("env", {}),
-      secrets_file: File.join(config.host_env_directory, "traefik", "traefik.env")
-  end
-
-  def make_env_directory
-    make_directory(env.secrets_directory)
-  end
-
-  def remove_env_file
-    [ :rm, "-f", env.secrets_file ]
-  end
-
-  private
-    def publish_args
-      argumentize "--publish", port unless config.traefik["publish"] == false
-    end
-
-    def label_args
-      argumentize "--label", labels
-    end
-
-    def env_args
-      env.args
-    end
-
-    def labels
-      DEFAULT_LABELS.merge(config.traefik["labels"] || {})
-    end
-
-    def image
-      config.traefik.fetch("image") { DEFAULT_IMAGE }
-    end
-
-    def docker_options_args
-      optionize(config.traefik["options"] || {})
-    end
-
-    def cmd_option_args
-      if args = config.traefik["args"]
-        optionize DEFAULT_ARGS.merge(args), with: "="
-      else
-        optionize DEFAULT_ARGS, with: "="
-      end
-    end
-
-    def host_port
-      config.traefik["host_port"] || CONTAINER_PORT
-    end
-end

lib/kamal/configuration.rb

@@ -6,7 +6,7 @@ require "erb"
 require "net/ssh/proxy/jump"
 
 class Kamal::Configuration
-  delegate :service, :image, :servers, :labels, :registry, :stop_wait_time, :hooks_path, :logging, to: :raw_config, allow_nil: true
+  delegate :service, :image, :port, :servers, :labels, :registry, :stop_wait_time, :hooks_path, :logging, to: :raw_config, allow_nil: true
   delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_reader :destination, :raw_config
@@ -88,7 +88,7 @@ class Kamal::Configuration
 
 
   def all_hosts
-    roles.flat_map(&:hosts).uniq
+    (roles + accessories).flat_map(&:hosts).uniq
   end
 
   def primary_host
@@ -107,16 +107,16 @@ class Kamal::Configuration
     raw_config.allow_empty_roles
   end
 
-  def traefik_roles
+  def proxy_roles
-    roles.select(&:running_traefik?)
+    roles.select(&:running_proxy?)
   end
 
-  def traefik_role_names
+  def proxy_role_names
-    traefik_roles.flat_map(&:name)
+    proxy_roles.flat_map(&:name)
   end
 
-  def traefik_hosts
+  def proxy_hosts
-    traefik_roles.flat_map(&:hosts).uniq
+    proxy_roles.flat_map(&:hosts).uniq
   end
 
   def repository
@@ -128,7 +128,11 @@ class Kamal::Configuration
   end
 
   def latest_image
-    "#{repository}:latest"
+    "#{repository}:#{latest_tag}"
+  end
+
+  def latest_tag
+    [ "latest", *destination ].join("-")
   end
 
   def service_with_version
@@ -170,8 +174,8 @@ class Kamal::Configuration
     Kamal::Configuration::Builder.new(config: self)
   end
 
-  def traefik
+  def proxy
-    raw_config.traefik || {}
+    Kamal::Configuration::Proxy.new(config: self)
   end
 
   def ssh
@@ -183,14 +187,6 @@ class Kamal::Configuration
   end
 
 
-  def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }.merge(raw_config.healthcheck || {})
-  end
-
-  def healthcheck_service
-    [ "healthcheck", service, destination ].compact.join("-")
-  end
-
   def readiness_delay
     raw_config.readiness_delay || 7
   end
@@ -222,13 +218,25 @@ class Kamal::Configuration
 
 
   def host_env_directory
-    "#{run_directory}/env"
+    File.join(run_directory, "env")
   end
 
   def env
     raw_config.env || {}
   end
 
+  def env_tags
+    @env_tags ||= if (tags = raw_config.env["tags"])
+      tags.collect { |name, config| Kamal::Configuration::Env::Tag.new(name, config: config) }
+    else
+      []
+    end
+  end
+
+  def env_tag(name)
+    env_tags.detect { |t| t.name == name.to_s }
+  end
+
 
   def valid?
     ensure_destination_if_required && ensure_required_keys_present && ensure_valid_kamal_version && ensure_retain_containers_valid && ensure_valid_service_name
@@ -248,8 +256,7 @@ class Kamal::Configuration
       sshkit: sshkit.to_h,
       builder: builder.to_h,
       accessories: raw_config.accessories,
-      logging: logging_args,
+      logging: logging_args
-      healthcheck: healthcheck
     }.compact
   end
 
@@ -297,7 +304,7 @@ class Kamal::Configuration
     end
 
     def ensure_valid_service_name
-      raise ArgumentError, "Service name can only include alphanumeric characters, hyphens, and underscores" unless raw_config[:service] =~ /^[a-z0-9_-]+$/
+      raise ArgumentError, "Service name can only include alphanumeric characters, hyphens, and underscores" unless raw_config[:service] =~ /^[a-z0-9_-]+$/i
 
       true
     end
@@ -324,7 +331,7 @@ class Kamal::Configuration
     def git_version
       @git_version ||=
         if Kamal::Git.used?
-          if Kamal::Git.uncommitted_changes.present? && !builder.git_archive?
+          if Kamal::Git.uncommitted_changes.present? && !builder.git_clone?
             uncommitted_suffix = "_uncommitted_#{SecureRandom.hex(8)}"
           end
           [ Kamal::Git.revision, uncommitted_suffix ].compact.join

lib/kamal/configuration/accessory.rb

@@ -134,7 +134,7 @@ class Kamal::Configuration::Accessory
     end
 
     def expand_host_path(host_path)
-      absolute_path?(host_path) ? host_path : "#{service_data_directory}/#{host_path}"
+      absolute_path?(host_path) ? host_path : File.join(service_data_directory, host_path)
     end
 
     def absolute_path?(path)

lib/kamal/configuration/builder.rb

@@ -3,6 +3,8 @@ class Kamal::Configuration::Builder
     @options = config.raw_config.builder || {}
     @image = config.image
     @server = config.registry["server"]
+    @service = config.service
+    @destination = config.destination
 
     valid?
   end
@@ -39,8 +41,12 @@ class Kamal::Configuration::Builder
     @options["dockerfile"] || "Dockerfile"
   end
 
+  def target
+    @options["target"]
+  end
+
   def context
-    @options["context"] || (git_archive? ? "-" : ".")
+    @options["context"] || "."
   end
 
   def local_arch
@@ -85,10 +91,23 @@ class Kamal::Configuration::Builder
     @options["ssh"]
   end
 
-  def git_archive?
+  def git_clone?
     Kamal::Git.used? && @options["context"].nil?
   end
 
+  def clone_directory
+    @clone_directory ||= File.join Dir.tmpdir, "kamal-clones", [ @service, pwd_sha ].compact.join("-")
+  end
+
+  def build_directory
+    @build_directory ||=
+      if git_clone?
+        File.join clone_directory, repo_basename, repo_relative_pwd
+      else
+        "."
+      end
+  end
+
   private
     def valid?
       if @options["cache"] && @options["cache"]["type"]
@@ -119,4 +138,16 @@ class Kamal::Configuration::Builder
     def cache_to_config_for_registry
       [ "type=registry", @options["cache"]&.fetch("options", nil), "ref=#{cache_image_ref}" ].compact.join(",")
     end
+
+    def repo_basename
+      File.basename(Kamal::Git.root)
+    end
+
+    def repo_relative_pwd
+      Dir.pwd.delete_prefix(Kamal::Git.root)
+    end
+
+    def pwd_sha
+      Digest::SHA256.hexdigest(Dir.pwd)[0..12]
+    end
 end

lib/kamal/configuration/env.rb

@@ -4,7 +4,7 @@ class Kamal::Configuration::Env
 
   def self.from_config(config:, secrets_file: nil)
     secrets_keys = config.fetch("secret", [])
-    clear = config.fetch("clear", config.key?("secret") ? {} : config)
+    clear = config.fetch("clear", config.key?("secret") || config.key?("tags") ? {} : config)
 
     new clear: clear, secrets_keys: secrets_keys, secrets_file: secrets_file
   end

lib/kamal/configuration/env/tag.rb

@@ -0,0 +1,12 @@
+class Kamal::Configuration::Env::Tag
+  attr_reader :name, :config
+
+  def initialize(name, config:)
+    @name = name
+    @config = config
+  end
+
+  def env
+    Kamal::Configuration::Env.from_config(config: config)
+  end
+end

lib/kamal/configuration/proxy.rb

@@ -0,0 +1,50 @@
+class Kamal::Configuration::Proxy
+  DEFAULT_HTTP_PORT = 80
+  DEFAULT_HTTPS_PORT = 443
+  DEFAULT_IMAGE = "basecamp/kamal-proxy:latest"
+
+  delegate :argumentize, :optionize, to: Kamal::Utils
+
+  def initialize(config:)
+    @options = config.raw_config.proxy || {}
+  end
+
+  def image
+    options.fetch("image", DEFAULT_IMAGE)
+  end
+
+  def debug?
+    !!options[:debug]
+  end
+
+  def http_port
+    options.fetch(:http_port, DEFAULT_HTTP_PORT)
+  end
+
+  def https_port
+    options.fetch(:http_port, DEFAULT_HTTPS_PORT)
+  end
+
+  def container_name
+    "kamal-proxy"
+  end
+
+  def docker_options_args
+    optionize(options.fetch("options", {}))
+  end
+
+  def publish_args
+    argumentize "--publish", [ *("#{http_port}:#{DEFAULT_HTTP_PORT}" if http_port), *("#{https_port}:#{DEFAULT_HTTPS_PORT}" if https_port) ]
+  end
+
+  def deploy_options
+    options.fetch(:deploy, {})
+  end
+
+  def deploy_command_args
+    optionize deploy_options
+  end
+
+  private
+    attr_accessor :options
+end

lib/kamal/configuration/role.rb

@@ -1,5 +1,4 @@
 class Kamal::Configuration::Role
-  CORD_FILE = "cord"
   delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_accessor :name
@@ -7,6 +6,7 @@ class Kamal::Configuration::Role
 
   def initialize(name, config:)
     @name, @config = name.inquiry, config
+    @tagged_hosts ||= extract_tagged_hosts_from_config
   end
 
   def primary_host
@@ -14,7 +14,11 @@ class Kamal::Configuration::Role
   end
 
   def hosts
-    @hosts ||= extract_hosts_from_config
+    tagged_hosts.keys
+  end
+
+  def env_tags(host)
+    tagged_hosts.fetch(host).collect { |tag| config.env_tag(tag) }
   end
 
   def cmd
@@ -30,7 +34,7 @@ class Kamal::Configuration::Role
   end
 
   def labels
-    default_labels.merge(traefik_labels).merge(custom_labels)
+    default_labels.merge(custom_labels)
   end
 
   def label_args
@@ -50,12 +54,13 @@ class Kamal::Configuration::Role
   end
 
 
-  def env
+  def env(host)
-    @env ||= base_env.merge(specialized_env)
+    @envs ||= {}
+    @envs[host] ||= [ base_env, specialized_env, *env_tags(host).map(&:env) ].reduce(:merge)
   end
 
-  def env_args
+  def env_args(host)
-    env.args
+    env(host).args
   end
 
   def asset_volume_args
@@ -63,37 +68,11 @@ class Kamal::Configuration::Role
   end
 
 
-  def health_check_args(cord: true)
+  def running_proxy?
-    if health_check_cmd.present?
+    if specializations["proxy"].nil?
-      if cord && uses_cord?
-        optionize({ "health-cmd" => health_check_cmd_with_cord, "health-interval" => health_check_interval })
-          .concat(cord_volume.docker_args)
-      else
-        optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval })
-      end
-    else
-      []
-    end
-  end
-
-  def health_check_cmd
-    health_check_options["cmd"] || http_health_check(port: health_check_options["port"], path: health_check_options["path"])
-  end
-
-  def health_check_cmd_with_cord
-    "(#{health_check_cmd}) && (stat #{cord_container_file} > /dev/null || exit 1)"
-  end
-
-  def health_check_interval
-    health_check_options["interval"] || "1s"
-  end
-
-
-  def running_traefik?
-    if specializations["traefik"].nil?
       primary?
     else
-      specializations["traefik"]
+      specializations["proxy"]
     end
   end
 
@@ -102,35 +81,6 @@ class Kamal::Configuration::Role
   end
 
 
-  def uses_cord?
-    running_traefik? && cord_volume && health_check_cmd.present?
-  end
-
-  def cord_host_directory
-    File.join config.run_directory_as_docker_volume, "cords", [ container_prefix, config.run_id ].join("-")
-  end
-
-  def cord_volume
-    if (cord = health_check_options["cord"])
-      @cord_volume ||= Kamal::Configuration::Volume.new \
-        host_path: File.join(config.run_directory, "cords", [ container_prefix, config.run_id ].join("-")),
-        container_path: cord
-    end
-  end
-
-  def cord_host_file
-    File.join cord_volume.host_path, CORD_FILE
-  end
-
-  def cord_container_directory
-    health_check_options.fetch("cord", nil)
-  end
-
-  def cord_container_file
-    File.join cord_volume.container_path, CORD_FILE
-  end
-
-
   def container_name(version = nil)
     [ container_prefix, version || config.version ].compact.join("-")
   end
@@ -145,7 +95,7 @@ class Kamal::Configuration::Role
   end
 
   def assets?
-    asset_path.present? && running_traefik?
+    asset_path.present? && running_proxy?
   end
 
   def asset_volume(version = nil)
@@ -164,7 +114,24 @@ class Kamal::Configuration::Role
   end
 
   private
-    attr_accessor :config
+    attr_accessor :config, :tagged_hosts
+
+    def extract_tagged_hosts_from_config
+      {}.tap do |tagged_hosts|
+        extract_hosts_from_config.map do |host_config|
+          if host_config.is_a?(Hash)
+            raise ArgumentError, "Multiple hosts found: #{host_config.inspect}" unless host_config.size == 1
+
+            host, tags = host_config.first
+            tagged_hosts[host] = Array(tags)
+          elsif host_config.is_a?(String) || host_config.is_a?(Symbol)
+            tagged_hosts[host_config] = []
+          else
+            raise ArgumentError, "Invalid host config: #{host_config.inspect}"
+          end
+        end
+      end
+    end
 
     def extract_hosts_from_config
       if config.servers.is_a?(Array)
@@ -179,27 +146,6 @@ class Kamal::Configuration::Role
       { "service" => config.service, "role" => name, "destination" => config.destination }
     end
 
-    def traefik_labels
-      if running_traefik?
-        {
-          # Setting a service property ensures that the generated service name will be consistent between versions
-          "traefik.http.services.#{traefik_service}.loadbalancer.server.scheme" => "http",
-
-          "traefik.http.routers.#{traefik_service}.rule" => "PathPrefix(`/`)",
-          "traefik.http.routers.#{traefik_service}.priority" => "2",
-          "traefik.http.middlewares.#{traefik_service}-retry.retry.attempts" => "5",
-          "traefik.http.middlewares.#{traefik_service}-retry.retry.initialinterval" => "500ms",
-          "traefik.http.routers.#{traefik_service}.middlewares" => "#{traefik_service}-retry@docker"
-        }
-      else
-        {}
-      end
-    end
-
-    def traefik_service
-      container_prefix
-    end
-
     def custom_labels
       Hash.new.tap do |labels|
         labels.merge!(config.labels) if config.labels.present?
@@ -225,16 +171,4 @@ class Kamal::Configuration::Role
         config: config.env,
         secrets_file: File.join(config.host_env_directory, "roles", "#{container_prefix}.env")
     end
-
-    def http_health_check(port:, path:)
-      "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present?
-    end
-
-    def health_check_options
-      @health_check_options ||= begin
-        options = specializations["healthcheck"] || {}
-        options = config.healthcheck.merge(options) if running_traefik?
-        options
-      end
-    end
 end

lib/kamal/env_file.rb

@@ -24,6 +24,13 @@ class Kamal::EnvFile
 
     # Escape a value to make it safe to dump in a docker file.
     def escape_docker_env_file_value(value)
+      # keep non-ascii(UTF-8) characters as it is
+      value.to_s.scan(/[\x00-\x7F]+|[^\x00-\x7F]+/).map do |part|
+        part.ascii_only? ? escape_docker_env_file_ascii_value(part) : part
+      end.join
+    end
+
+    def escape_docker_env_file_ascii_value(value)
       # Doublequotes are treated literally in docker env files
       # so remove leading and trailing ones and unescape any others
       value.to_s.dump[1..-2].gsub(/\\"/, "\"")

lib/kamal/git.rb

@@ -16,4 +16,8 @@ module Kamal::Git
   def uncommitted_changes
     `git status --porcelain`.strip
   end
+
+  def root
+    `git rev-parse --show-toplevel`.strip
+  end
 end

lib/kamal/sshkit_with_ext.rb

@@ -103,3 +103,39 @@ class SSHKit::Backend::Netssh
 
   prepend LimitConcurrentStartsInstance
 end
+
+class SSHKit::Runner::Parallel
+  # SSHKit joins the threads in sequence and fails on the first error it encounters, which means that we wait threads
+  # before the first failure to complete but not for ones after.
+  #
+  # We'll patch it to wait for them all to complete, and to record all the threads that errored so we can see when a
+  # problem occurs on multiple hosts.
+  module CompleteAll
+    def execute
+      threads = hosts.map do |host|
+        Thread.new(host) do |h|
+          backend(h, &block).run
+        rescue ::StandardError => e
+          e2 = SSHKit::Runner::ExecuteError.new e
+          raise e2, "Exception while executing #{host.user ? "as #{host.user}@" : "on host "}#{host}: #{e.message}"
+        end
+      end
+
+      exceptions = []
+      threads.each do |t|
+        begin
+          t.join
+        rescue SSHKit::Runner::ExecuteError => e
+          exceptions << e
+        end
+      end
+      if exceptions.one?
+        raise exceptions.first
+      elsif exceptions.many?
+        raise exceptions.first, [ "Exceptions on #{exceptions.count} hosts:", exceptions.map(&:message) ].join("\n")
+      end
+    end
+  end
+
+  prepend CompleteAll
+end

lib/kamal/utils.rb

@@ -66,12 +66,15 @@ module Kamal::Utils
     Array(filters).select do |filter|
       matches += Array(items).select do |item|
         # Only allow * for a wildcard
-        pattern = Regexp.escape(filter).gsub('\*', ".*")
         # items are roles or hosts
-        (item.respond_to?(:name) ? item.name : item).match(/^#{pattern}$/)
+        File.fnmatch(filter, item.to_s, File::FNM_EXTGLOB)
       end
     end
 
-    matches
+    matches.uniq
+  end
+
+  def stable_sort!(elements, &block)
+    elements.sort_by!.with_index { |element, index| [ block.call(element), index ] }
   end
 end

lib/kamal/version.rb

@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "1.4.0"
+  VERSION = "1.5.2"
 end

test/cli/accessory_test.rb

@@ -76,7 +76,10 @@ class CliAccessoryTest < CliTestCase
   end
 
   test "details" do
-    assert_match "docker ps --filter label=service=app-mysql", run_command("details", "mysql")
+    run_command("details", "mysql").tap do |output|
+      assert_match "docker ps --filter label=service=app-mysql", output
+      assert_match "Accessory mysql Host: 1.1.1.3", output
+    end
   end
 
   test "details with non-existent accessory" do
@@ -85,6 +88,8 @@ class CliAccessoryTest < CliTestCase
 
   test "details with all" do
     run_command("details", "all").tap do |output|
+      assert_match "Accessory mysql Host: 1.1.1.3", output
+      assert_match "Accessory redis Host: 1.1.1.2", output
       assert_match "docker ps --filter label=service=app-mysql", output
       assert_match "docker ps --filter label=service=app-redis", output
     end

test/cli/app_test.rb

@@ -11,7 +11,6 @@ class CliAppTest < CliTestCase
   end
 
   test "boot will rename if same version is already running" do
-    Object.any_instance.stubs(:sleep)
     run_command("details") # Preheat Kamal const
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -19,20 +18,13 @@ class CliAppTest < CliTestCase
       .returns("12345678") # running version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
-      .returns("running") # health check
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
       .returns("123") # old version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'", "|", :sed, "-e", "'s/\\/tcp$//'")
-      .returns("cordfile") # old version
+      .returns("172.1.0.2:80")
-
+      .at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-      .returns("unhealthy") # old version unhealthy
 
     run_command("boot").tap do |output|
       assert_match /Renaming container .* to .* as already deployed on 1.1.1.1/, output # Rename
@@ -54,33 +46,27 @@ class CliAppTest < CliTestCase
     run_command("boot", config: :with_boot_strategy)
   end
 
-  test "boot errors leave lock in place" do
+  test "boot errors don't leave lock in place" do
     Kamal::Cli::App.any_instance.expects(:using_version).raises(RuntimeError)
 
     assert_not KAMAL.holding_lock?
     assert_raises(RuntimeError) do
       stderred { run_command("boot") }
     end
-    assert KAMAL.holding_lock?
+    assert_not KAMAL.holding_lock?
   end
 
   test "boot with assets" do
-    Object.any_instance.stubs(:sleep)
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
       .returns("12345678") # running version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
-      .returns("running") # health check
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
       .returns("123").twice # old version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'", "|", :sed, "-e", "'s/\\/tcp$//'")
-      .returns("") # old version
+      .returns("172.1.0.2:80").at_least_once
 
     run_command("boot", config: :with_assets).tap do |output|
       assert_match "docker tag dhh/app:latest dhh/app:latest", output
@@ -92,7 +78,91 @@ class CliAppTest < CliTestCase
     end
   end
 
+  test "boot with host tags" do
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678") # running version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'", "|", :sed, "-e", "'s/\\/tcp$//'")
+      .returns("172.1.0.2:80").at_least_once
+
+    run_command("boot", config: :with_env_tags).tap do |output|
+      assert_match "docker tag dhh/app:latest dhh/app:latest", output
+      assert_match %r{docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} -e KAMAL_CONTAINER_NAME="app-web-latest" -e KAMAL_VERSION="latest" --env-file .kamal/env/roles/app-web.env --env TEST="root" --env EXPERIMENT="disabled" --env SITE="site1"}, output
+      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+    end
+  end
+
+  test "boot with web barrier opened" do
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'", "|", :sed, "-e", "'s/\\/tcp$//'")
+      .returns("172.1.0.2:80").at_least_once
+
+    run_command("boot", config: :with_roles, host: nil).tap do |output|
+      assert_match "Waiting for the first healthy web container before booting workers on 1.1.1.3...", output
+      assert_match "Waiting for the first healthy web container before booting workers on 1.1.1.4...", output
+      assert_match "First web container is healthy, booting workers on 1.1.1.3", output
+      assert_match "First web container is healthy, booting workers on 1.1.1.4", output
+    end
+  end
+
+  test "boot with web barrier closed" do
+    Thread.report_on_exception = false
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("abcdef123456")
+      .twice # web container id
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("abcdef123456")
+      .twice # worker container id
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with { |*args| args[0..1] == [ :sh, "-c" ] }.returns("123").at_least_once
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'", "|", :sed, "-e", "'s/\\/tcp$//'")
+      .returns("172.1.0.2:80")
+      .at_least_once
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute).returns("")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, :exec, "kamal-proxy", "kamal-proxy", :deploy, "app-web", "--target", "\"172.1.0.2:80\"").raises(SSHKit::Command::Failed, "Deploy failed").at_least_once
+
+    stderred do
+      run_command("boot", config: :with_roles, host: nil, allowed_error_message: "Deploy failed").tap do |output|
+        assert_match "Waiting for the first healthy web container before booting workers on 1.1.1.3...", output
+        assert_match "Waiting for the first healthy web container before booting workers on 1.1.1.4...", output
+        assert_match "First web container is unhealthy, not booting workers on 1.1.1.3", output
+        assert_match "First web container is unhealthy, not booting workers on 1.1.1.4", output
+      end
+    end
+  ensure
+    Thread.report_on_exception = true
+  end
+
   test "start" do
+    # SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("123") # current version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'", "|", :sed, "-e", "'s/\\/tcp$//'")
+      .returns("172.1.0.2:80")
+      .at_least_once
+
     run_command("start").tap do |output|
       assert_match "docker start app-web-999", output
     end
@@ -100,14 +170,18 @@ class CliAppTest < CliTestCase
 
   test "stop" do
     run_command("stop").tap do |output|
-      assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop", output
+      assert_match "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop", output
     end
   end
 
   test "stale_containers" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
-      .returns("12345678\n87654321")
+      .returns("12345678\n87654321\n")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("12345678\n")
 
     run_command("stale_containers").tap do |output|
       assert_match /Detected stale container for role web with version 87654321/, output
@@ -117,7 +191,11 @@ class CliAppTest < CliTestCase
   test "stop stale_containers" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
-      .returns("12345678\n87654321")
+      .returns("12345678\n87654321\n")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("12345678\n")
 
     run_command("stale_containers", "--stop").tap do |output|
       assert_match /Stopping stale container for role web with version 87654321/, output
@@ -133,7 +211,7 @@ class CliAppTest < CliTestCase
 
   test "remove" do
     run_command("remove").tap do |output|
-      assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop")}/, output
+      assert_match /#{Regexp.escape("sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop")}/, output
       assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output
       assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output
     end
@@ -165,7 +243,7 @@ class CliAppTest < CliTestCase
 
   test "exec with reuse" do
     run_command("exec", "--reuse", "ruby -v").tap do |output|
-      assert_match "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done", output # Get current version
+      assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output # Get current version
       assert_match "docker exec app-web-999 ruby -v", output
     end
   end
@@ -184,7 +262,7 @@ class CliAppTest < CliTestCase
       .with("ssh -t root@1.1.1.1 -p 22 'docker exec -it app-web-999 ruby -v'")
     run_command("exec", "-i", "--reuse", "ruby -v").tap do |output|
       assert_match "Get current version of running container...", output
-      assert_match "Running docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done on 1.1.1.1", output
+      assert_match "Running /usr/bin/env sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done on 1.1.1.1", output
       assert_match "Launching interactive command with version 999 via SSH from existing container on 1.1.1.1...", output
     end
   end
@@ -203,47 +281,61 @@ class CliAppTest < CliTestCase
 
   test "logs" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest| xargs docker logs --timestamps --tail 10 2>&1'")
+      .with("ssh -t root@1.1.1.1 'sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1| xargs docker logs --timestamps --tail 10 2>&1'")
 
-    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --tail 100 2>&1", run_command("logs")
+    assert_match "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --tail 100 2>&1", run_command("logs")
   end
 
   test "logs with follow" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 -p 22 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
+      .with("ssh -t root@1.1.1.1 -p 22 'sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
 
-    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
+    assert_match "sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
   end
 
   test "version" do
     run_command("version").tap do |output|
-      assert_match "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done", output
+      assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output
     end
   end
 
 
   test "version through main" do
     stdouted { Kamal::Cli::Main.start([ "app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1" ]) }.tap do |output|
-      assert_match "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done", output
+      assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output
+    end
+  end
+
+  test "long hostname" do
+    stub_running
+
+    hostname = "this-hostname-is-really-unacceptably-long-to-be-honest.example.com"
+
+    stdouted { Kamal::Cli::App.start([ "boot", "-c", "test/fixtures/deploy_with_uncommon_hostnames.yml", "--hosts", hostname ]) }.tap do |output|
+      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname this-hostname-is-really-unacceptably-long-to-be-hon-[0-9a-f]{12} /, output
+    end
+  end
+
+  test "hostname is trimmed if will end with a period" do
+    stub_running
+
+    hostname = "this-hostname-with-random-part-is-too-long.example.com"
+
+    stdouted { Kamal::Cli::App.start([ "boot", "-c", "test/fixtures/deploy_with_uncommon_hostnames.yml", "--hosts", hostname ]) }.tap do |output|
+      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname this-hostname-with-random-part-is-too-long.example-[0-9a-f]{12} /, output
     end
   end
 
   private
-    def run_command(*command, config: :with_accessories)
+    def run_command(*command, config: :with_accessories, host: "1.1.1.1", allowed_error_message: nil)
-      stdouted { Kamal::Cli::App.start([ *command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1" ]) }
+      stdouted do
+        Kamal::Cli::App.start([ *command, "-c", "test/fixtures/deploy_#{config}.yml", *([ "--hosts", host ] if host) ])
+      rescue SSHKit::Runner::ExecuteError => e
+        raise e unless allowed_error_message && e.message.include?(allowed_error_message)
+      end
     end
 
     def stub_running
-      Object.any_instance.stubs(:sleep)
-
       SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
-
-      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-        .returns("running") # health check
-
-      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-        .returns("unhealthy") # health check
     end
 end

test/cli/build_test.rb

@@ -9,32 +9,137 @@ class CliBuildTest < CliTestCase
   end
 
   test "push" do
+    with_build_directory do |build_directory|
       Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
       hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" }
 
-    run_command("push").tap do |output|
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :"rev-parse", :HEAD)
+        .returns(Kamal::Git.revision)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :status, "--porcelain")
+        .returns("")
+
+      run_command("push", "--verbose").tap do |output|
+        assert_hook_ran "pre-build", output, **hook_variables
+        assert_match /Cloning repo into build directory/, output
+        assert_match /git -C #{Dir.tmpdir}\/kamal-clones\/app-#{pwd_sha} clone #{Dir.pwd}/, output
+        assert_match /docker --version && docker buildx version/, output
+        assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
+      end
+    end
+  end
+
+  test "push resetting clone" do
+    with_build_directory do |build_directory|
+      stub_setup
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd)
+        .raises(SSHKit::Command::Failed.new("fatal: destination path 'kamal' already exists and is not an empty directory"))
+        .then
+        .returns(true)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :remote, "set-url", :origin, Dir.pwd)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :fetch, :origin)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :reset, "--hard", Kamal::Git.revision)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :clean, "-fdx")
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with(:docker, :buildx, :build, "--push", "--platform", "linux/amd64,linux/arm64", "--builder", "kamal-app-multiarch", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".")
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :"rev-parse", :HEAD)
+        .returns(Kamal::Git.revision)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :status, "--porcelain")
+        .returns("")
+
+      run_command("push", "--verbose").tap do |output|
+        assert_match /Cloning repo into build directory/, output
+        assert_match /Resetting local clone/, output
+      end
+    end
+  end
+
+  test "push without clone" do
+    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+    hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" }
+
+    run_command("push", "--verbose", fixture: :without_clone).tap do |output|
+      assert_no_match /Cloning repo into build directory/, output
       assert_hook_ran "pre-build", output, **hook_variables
       assert_match /docker --version && docker buildx version/, output
-      assert_match /git archive -tar HEAD | docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile - as .*@localhost/, output
+      assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile . as .*@localhost/, output
+    end
+  end
+
+  test "push with corrupt clone" do
+    with_build_directory do |build_directory|
+      stub_setup
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd)
+        .raises(SSHKit::Command::Failed.new("fatal: destination path 'kamal' already exists and is not an empty directory"))
+        .then
+        .returns(true)
+        .twice
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :remote, "set-url", :origin, Dir.pwd)
+        .raises(SSHKit::Command::Failed.new("fatal: not a git repository"))
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :"rev-parse", :HEAD)
+        .returns(Kamal::Git.revision)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :status, "--porcelain")
+        .returns("")
+
+      Dir.stubs(:chdir)
+
+      run_command("push", "--verbose") do |output|
+        assert_match /Cloning repo into build directory `#{build_directory}`\.\.\..*Cloning repo into build directory `#{build_directory}`\.\.\./, output
+        assert_match "Resetting local clone as `#{build_directory}` already exists...", output
+        assert_match "Error preparing clone: Failed to clone repo: fatal: not a git repository, deleting and retrying...", output
+      end
     end
   end
 
   test "push without builder" do
+    with_build_directory do |build_directory|
       stub_setup
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
         .with(:docker, "--version", "&&", :docker, :buildx, "version")
 
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
         .with(:docker, :buildx, :create, "--use", "--name", "kamal-app-multiarch")
 
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
-      .with { |*args| p args[0..6]; args[0..6] == [ :git, :archive, "--format=tar", :HEAD, "|", :docker, :buildx ] }
+        .with { |*args| args[0..1] == [ :docker, :buildx ] }
         .raises(SSHKit::Command::Failed.new("no builder"))
         .then
         .returns(true)
 
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with { |*args| args.first.start_with?("git") }
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :"rev-parse", :HEAD)
+        .returns(Kamal::Git.revision)
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:git, "-C", anything, :status, "--porcelain")
+        .returns("")
+
       run_command("push").tap do |output|
-      assert_match /Missing compatible builder, so creating a new one first/, output
+        assert_match /WARN Missing compatible builder, so creating a new one first/, output
+      end
     end
   end
 
@@ -51,7 +156,8 @@ class CliBuildTest < CliTestCase
   test "push pre-build hook failure" do
     fail_hook("pre-build")
 
-    assert_raises(Kamal::Cli::HookError) { run_command("push") }
+    error = assert_raises(Kamal::Cli::HookError) { run_command("push") }
+    assert_equal "Hook `pre-build` failed:\nfailed", error.message
 
     assert @executions.none? { |args| args[0..2] == [ :docker, :buildx, :build ] }
   end
@@ -78,6 +184,14 @@ class CliBuildTest < CliTestCase
     end
   end
 
+  test "create remote with custom ports" do
+    run_command("create", fixture: :with_remote_builder_and_custom_ports).tap do |output|
+      assert_match "Running /usr/bin/env true on 1.1.1.5", output
+      assert_match "docker context create kamal-app-native-remote-amd64 --description 'kamal-app-native-remote amd64 native host' --docker 'host=ssh://app@1.1.1.5:2122'", output
+      assert_match "docker buildx create --name kamal-app-native-remote kamal-app-native-remote-amd64 --platform linux/amd64", output
+    end
+  end
+
   test "create with error" do
     stub_setup
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
@@ -117,4 +231,17 @@ class CliBuildTest < CliTestCase
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
         .with { |*args| args[0..1] == [ :docker, :buildx ] }
     end
+
+    def with_build_directory
+      build_directory = File.join Dir.tmpdir, "kamal-clones", "app-#{pwd_sha}", "kamal"
+      FileUtils.mkdir_p build_directory
+      FileUtils.touch File.join build_directory, "Dockerfile"
+      yield build_directory + "/"
+    ensure
+      FileUtils.rm_rf build_directory
+    end
+
+    def pwd_sha
+      Digest::SHA256.hexdigest(Dir.pwd)[0..12]
+    end
 end

test/cli/cli_test_case.rb

@@ -38,7 +38,7 @@ class CliTestCase < ActiveSupport::TestCase
         .with { |arg1, arg2| arg1 == :rm && arg2 == ".kamal/locks/app/details" }
     end
 
-    def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: nil)
+    def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: false)
       performer = `whoami`.strip
 
       assert_match "Running the #{hook} hook...\n", output
@@ -52,7 +52,7 @@ class CliTestCase < ActiveSupport::TestCase
         KAMAL_HOSTS=\"#{hosts}\"\s
         KAMAL_COMMAND=\"#{command}\"\s
         #{"KAMAL_SUBCOMMAND=\\\"#{subcommand}\\\"\\s" if subcommand}
-        #{"KAMAL_RUNTIME=\\\"#{runtime}\\\"\\s" if runtime}
+        #{"KAMAL_RUNTIME=\\\"\\d+\\\"\\s" if runtime}
         ;\s/usr/bin/env\s\.kamal/hooks/#{hook} }x
 
       assert_match expected, output

test/cli/env_test.rb

@@ -4,14 +4,11 @@ class CliEnvTest < CliTestCase
   test "push" do
     run_command("push").tap do |output|
       assert_match "Running /usr/bin/env mkdir -p .kamal/env/roles on 1.1.1.1", output
-      assert_match "Running /usr/bin/env mkdir -p .kamal/env/traefik on 1.1.1.1", output
       assert_match "Running /usr/bin/env mkdir -p .kamal/env/accessories on 1.1.1.1", output
       assert_match "Running /usr/bin/env mkdir -p .kamal/env/roles on 1.1.1.1", output
-      assert_match "Running /usr/bin/env mkdir -p .kamal/env/traefik on 1.1.1.2", output
       assert_match "Running /usr/bin/env mkdir -p .kamal/env/accessories on 1.1.1.1", output
       assert_match ".kamal/env/roles/app-web.env", output
       assert_match ".kamal/env/roles/app-workers.env", output
-      assert_match ".kamal/env/traefik/traefik.env", output
       assert_match ".kamal/env/accessories/app-redis.env", output
     end
   end
@@ -22,8 +19,6 @@ class CliEnvTest < CliTestCase
       assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-web.env on 1.1.1.2", output
       assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-workers.env on 1.1.1.3", output
       assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-workers.env on 1.1.1.4", output
-      assert_match "Running /usr/bin/env rm -f .kamal/env/traefik/traefik.env on 1.1.1.1", output
-      assert_match "Running /usr/bin/env rm -f .kamal/env/traefik/traefik.env on 1.1.1.2", output
       assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-redis.env on 1.1.1.1", output
       assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-redis.env on 1.1.1.2", output
       assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-mysql.env on 1.1.1.3", output

test/cli/healthcheck_test.rb

@@ -1,82 +0,0 @@
-require_relative "cli_test_case"
-
-class CliHealthcheckTest < CliTestCase
-  test "perform" do
-    # Prevent expected failures from outputting to terminal
-    Thread.report_on_exception = false
-
-    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
-    Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
-
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--env-file", ".kamal/env/roles/app-web.env", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false)
-
-    # Fail twice to test retry logic
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-      .returns("starting")
-      .then
-      .returns("unhealthy")
-      .then
-      .returns("healthy")
-
-    run_command("perform").tap do |output|
-      assert_match "container not ready (starting), retrying in 1s (attempt 1/7)...", output
-      assert_match "container not ready (unhealthy), retrying in 2s (attempt 2/7)...", output
-      assert_match "Container is healthy!", output
-    end
-  end
-
-  test "perform failing to become healthy" do
-    # Prevent expected failures from outputting to terminal
-    Thread.report_on_exception = false
-
-    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
-
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--env-file", ".kamal/env/roles/app-web.env", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false)
-
-    # Continually report unhealthy
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-      .returns("unhealthy")
-
-    # Capture logs when failing
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :logs, "--tail", 50, "2>&1")
-      .returns("some log output")
-
-    # Capture container health log when failing
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_pretty_json)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :inspect, "--format",  "'{{json .State.Health}}'")
-      .returns('{"Status":"unhealthy","Log":[{"ExitCode": 1,"Output": "/bin/sh: 1: curl: not found\n"}]}"')
-
-    exception = assert_raises do
-      run_command("perform")
-    end
-    assert_match "container not ready (unhealthy)", exception.message
-  end
-
-  test "raises an exception if primary does not have traefik" do
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).never
-
-    exception = assert_raises do
-      run_command("perform", config_file: "test/fixtures/deploy_workers_only.yml")
-    end
-
-    assert_equal "The primary host is not configured to run Traefik", exception.message
-  end
-
-  private
-    def run_command(*command, config_file: "test/fixtures/deploy_with_accessories.yml")
-      stdouted { Kamal::Cli::Healthcheck.start([ *command, "-c", config_file ]) }
-    end
-end

test/cli/main_test.rb

@@ -5,13 +5,13 @@ class CliMainTest < CliTestCase
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:main:envify", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:deploy)
 
     run_command("setup").tap do |output|
       assert_match /Ensure Docker is installed.../, output
-      assert_match /Push env files.../, output
+      assert_match /Evaluate and push env files.../, output
     end
   end
 
@@ -19,26 +19,24 @@ class CliMainTest < CliTestCase
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:main:envify", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
     # deploy
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
     run_command("setup", "--skip_push").tap do |output|
       assert_match /Ensure Docker is installed.../, output
-      assert_match /Push env files.../, output
+      assert_match /Evaluate and push env files.../, output
       # deploy
       assert_match /Acquiring the deploy lock/, output
       assert_match /Log into image registry/, output
       assert_match /Pull app image/, output
-      assert_match /Ensure Traefik is running/, output
+      assert_match /Ensure proxy is running/, output
-      assert_match /Ensure app can pass healthcheck/, output
       assert_match /Detect stale containers/, output
       assert_match /Prune old containers and images/, output
       assert_match /Releasing the deploy lock/, output
@@ -46,12 +44,11 @@ class CliMainTest < CliTestCase
   end
 
   test "deploy" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
@@ -59,16 +56,15 @@ class CliMainTest < CliTestCase
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "deploy" }
 
-    run_command("deploy").tap do |output|
+    run_command("deploy", "--verbose").tap do |output|
       assert_hook_ran "pre-connect", output, **hook_variables
       assert_match /Log into image registry/, output
       assert_match /Build and push app image/, output
       assert_hook_ran "pre-deploy", output, **hook_variables
-      assert_match /Ensure Traefik is running/, output
+      assert_match /Ensure proxy is running/, output
-      assert_match /Ensure app can pass healthcheck/, output
       assert_match /Detect stale containers/, output
       assert_match /Prune old containers and images/, output
-      assert_hook_ran "post-deploy", output, **hook_variables, runtime: 0
+      assert_hook_ran "post-deploy", output, **hook_variables, runtime: true
     end
   end
 
@@ -77,8 +73,7 @@ class CliMainTest < CliTestCase
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
@@ -87,8 +82,7 @@ class CliMainTest < CliTestCase
       assert_match /Acquiring the deploy lock/, output
       assert_match /Log into image registry/, output
       assert_match /Pull app image/, output
-      assert_match /Ensure Traefik is running/, output
+      assert_match /Ensure proxy is running/, output
-      assert_match /Ensure app can pass healthcheck/, output
       assert_match /Detect stale containers/, output
       assert_match /Prune old containers and images/, output
       assert_match /Releasing the deploy lock/, output
@@ -98,6 +92,9 @@ class CliMainTest < CliTestCase
   test "deploy when locked" do
     Thread.report_on_exception = false
 
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+    Dir.stubs(:chdir)
+
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }
 
@@ -111,6 +108,14 @@ class CliMainTest < CliTestCase
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_debug)
       .with(:stat, ".kamal/locks/app", ">", "/dev/null", "&&", :cat, ".kamal/locks/app/details", "|", :base64, "-d")
 
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:git, "-C", anything, :"rev-parse", :HEAD)
+      .returns(Kamal::Git.revision)
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:git, "-C", anything, :status, "--porcelain")
+      .returns("")
+
     assert_raises(Kamal::Cli::LockError) do
       run_command("deploy")
     end
@@ -119,6 +124,9 @@ class CliMainTest < CliTestCase
   test "deploy error when locking" do
     Thread.report_on_exception = false
 
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+    Dir.stubs(:chdir)
+
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }
 
@@ -129,6 +137,14 @@ class CliMainTest < CliTestCase
       .with { |*arg| arg[0..1] == [ :mkdir, ".kamal/locks/app" ] }
       .raises(SocketError, "getaddrinfo: nodename nor servname provided, or not known")
 
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:git, "-C", anything, :"rev-parse", :HEAD)
+      .returns(Kamal::Git.revision)
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:git, "-C", anything, :status, "--porcelain")
+      .returns("")
+
     assert_raises(SSHKit::Runner::ExecuteError) do
       run_command("deploy")
     end
@@ -153,8 +169,7 @@ class CliMainTest < CliTestCase
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
@@ -164,28 +179,12 @@ class CliMainTest < CliTestCase
     end
   end
 
-  test "deploy without healthcheck if primary host doesn't have traefik" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_workers_only.yml", "version" => "999", "skip_hooks" => false }
-
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options).never
-
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
-
-    run_command("deploy", config_file: "deploy_workers_only")
-  end
-
   test "deploy with missing secrets" do
     invoke_options = { "config_file" => "test/fixtures/deploy_with_secrets.yml", "version" => "999", "skip_hooks" => false }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
@@ -194,10 +193,9 @@ class CliMainTest < CliTestCase
   end
 
   test "redeploy" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
 
@@ -205,13 +203,12 @@ class CliMainTest < CliTestCase
 
     hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "redeploy" }
 
-    run_command("redeploy").tap do |output|
+    run_command("redeploy", "--verbose").tap do |output|
       assert_hook_ran "pre-connect", output, **hook_variables
       assert_match /Build and push app image/, output
       assert_hook_ran "pre-deploy", output, **hook_variables
       assert_match /Running the pre-deploy hook.../, output
-      assert_match /Ensure app can pass healthcheck/, output
+      assert_hook_ran "post-deploy", output, **hook_variables, runtime: true
-      assert_hook_ran "post-deploy", output, **hook_variables, runtime: "0"
     end
   end
 
@@ -219,13 +216,11 @@ class CliMainTest < CliTestCase
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
 
     run_command("redeploy", "--skip_push").tap do |output|
       assert_match /Pull app image/, output
-      assert_match /Ensure app can pass healthcheck/, output
     end
   end
 
@@ -241,7 +236,6 @@ class CliMainTest < CliTestCase
   end
 
   test "rollback good version" do
-    Object.any_instance.stubs(:sleep)
     [ "web", "workers" ].each do |role|
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
         .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false)
@@ -250,47 +244,38 @@ class CliMainTest < CliTestCase
         .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet")
         .returns("version-to-rollback\n").at_least_once
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-#{role}-}; done", raise_on_non_zero_exit: false)
+        .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=#{role} --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=#{role} --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-#{role}-}; done", raise_on_non_zero_exit: false)
         .returns("version-to-rollback\n").at_least_once
-      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-        .returns("running").at_least_once # health check
     end
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'", "|", :sed, "-e", "'s/\\/tcp$//'")
-      .returns("corddirectory").at_least_once # health check
+      .returns("172.1.0.2:80").at_least_once
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-version-to-rollback$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-      .returns("unhealthy").at_least_once # health check
 
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     hook_variables = { version: 123, service_version: "app@123", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "rollback" }
 
-    run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
+    run_command("rollback", "--verbose", "123", config_file: "deploy_with_accessories").tap do |output|
       assert_hook_ran "pre-deploy", output, **hook_variables
       assert_match "docker tag dhh/app:123 dhh/app:latest", output
       assert_match "docker run --detach --restart unless-stopped --name app-web-123", output
       assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running"
-      assert_hook_ran "post-deploy", output, **hook_variables, runtime: "0"
+      assert_hook_ran "post-deploy", output, **hook_variables, runtime: true
     end
   end
 
   test "rollback without old version" do
     Kamal::Cli::Main.any_instance.stubs(:container_available?).returns(true)
 
-    Kamal::Cli::Healthcheck::Poller.stubs(:sleep)
-
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
       .returns("").at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
       .returns("").at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'", "|", :sed, "-e", "'s/\\/tcp$//'")
-      .returns("running").at_least_once # health check
+      .returns("127.1.0.4:80").at_least_once
 
     run_command("rollback", "123").tap do |output|
       assert_match "docker run --detach --restart unless-stopped --name app-web-123", output
@@ -299,7 +284,7 @@ class CliMainTest < CliTestCase
   end
 
   test "details" do
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:details")
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:details")
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:details")
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:details", [ "all" ])
 
@@ -427,6 +412,7 @@ class CliMainTest < CliTestCase
   end
 
   test "envify" do
+    Pathname.any_instance.expects(:exist?).returns(true).times(3)
     File.expects(:read).with(".env.erb").returns("HELLO=<%= 'world' %>")
     File.expects(:write).with(".env", "HELLO=world", perm: 0600)
 
@@ -441,6 +427,7 @@ class CliMainTest < CliTestCase
       <% end  -%>
     EOF
 
+    Pathname.any_instance.expects(:exist?).returns(true).times(3)
     File.expects(:read).with(".env.erb").returns(file.strip)
     File.expects(:write).with(".env", "HELLO=world\nKEY=value\n", perm: 0600)
 
@@ -448,6 +435,7 @@ class CliMainTest < CliTestCase
   end
 
   test "envify with destination" do
+    Pathname.any_instance.expects(:exist?).returns(true).times(4)
     File.expects(:read).with(".env.world.erb").returns("HELLO=<%= 'world' %>")
     File.expects(:write).with(".env.world", "HELLO=world", perm: 0600)
 
@@ -455,6 +443,7 @@ class CliMainTest < CliTestCase
   end
 
   test "envify with skip_push" do
+    Pathname.any_instance.expects(:exist?).returns(true).times(1)
     File.expects(:read).with(".env.erb").returns("HELLO=<%= 'world' %>")
     File.expects(:write).with(".env", "HELLO=world", perm: 0600)
 
@@ -464,9 +453,9 @@ class CliMainTest < CliTestCase
 
   test "remove with confirmation" do
     run_command("remove", "-y", config_file: "deploy_with_accessories").tap do |output|
-      assert_match /docker container stop traefik/, output
+      assert_match /docker container stop kamal-proxy/, output
-      assert_match /docker container prune --force --filter label=org.opencontainers.image.title=Traefik/, output
+      assert_match /docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy/, output
-      assert_match /docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik/, output
+      assert_match /docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy/, output
 
       assert_match /docker ps --quiet --filter label=service=app | xargs docker stop/, output
       assert_match /docker container prune --force --filter label=service=app/, output

test/cli/proxy_test.rb

@@ -0,0 +1,116 @@
+require_relative "cli_test_case"
+
+class CliProxyTest < CliTestCase
+  test "boot" do
+    run_command("boot").tap do |output|
+      assert_match "docker login", output
+      assert_match "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}", output
+    end
+  end
+
+  test "reboot" do
+    Kamal::Commands::Registry.any_instance.expects(:login).twice
+
+    run_command("reboot", "-y").tap do |output|
+      assert_match "docker container stop kamal-proxy", output
+      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy", output
+      assert_match "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}", output
+    end
+  end
+
+  test "reboot --rolling" do
+    run_command("reboot", "--rolling", "-y").tap do |output|
+      assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.1", output
+    end
+  end
+
+  test "start" do
+    run_command("start").tap do |output|
+      assert_match "docker container start kamal-proxy", output
+    end
+  end
+
+  test "stop" do
+    run_command("stop").tap do |output|
+      assert_match "docker container stop kamal-proxy", output
+    end
+  end
+
+  test "restart" do
+    Kamal::Cli::Proxy.any_instance.expects(:stop)
+    Kamal::Cli::Proxy.any_instance.expects(:start)
+
+    run_command("restart")
+  end
+
+  test "details" do
+    run_command("details").tap do |output|
+      assert_match "docker ps --filter name=^kamal-proxy$", output
+    end
+  end
+
+  test "logs" do
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture)
+      .with(:docker, :logs, "kamal-proxy", " --tail 100", "--timestamps", "2>&1")
+      .returns("Log entry")
+
+    run_command("logs").tap do |output|
+      assert_match "Proxy Host: 1.1.1.1", output
+      assert_match "Log entry", output
+    end
+  end
+
+  test "logs with follow" do
+    SSHKit::Backend::Abstract.any_instance.stubs(:exec)
+      .with("ssh -t root@1.1.1.1 -p 22 'docker logs kamal-proxy --timestamps --tail 10 --follow 2>&1'")
+
+    assert_match "docker logs kamal-proxy --timestamps --tail 10 --follow", run_command("logs", "--follow")
+  end
+
+  test "remove" do
+    Kamal::Cli::Proxy.any_instance.expects(:stop)
+    Kamal::Cli::Proxy.any_instance.expects(:remove_container)
+    Kamal::Cli::Proxy.any_instance.expects(:remove_image)
+
+    run_command("remove")
+  end
+
+  test "remove_container" do
+    run_command("remove_container").tap do |output|
+      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy", output
+    end
+  end
+
+  test "remove_image" do
+    run_command("remove_image").tap do |output|
+      assert_match "docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy", output
+    end
+  end
+
+  test "update" do
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'", "|", :sed, "-e", "'s/\\/tcp$//'")
+      .returns("172.1.0.2:80")
+      .at_least_once
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with { |*args| args[0..1] == [ :sh, "-c" ] }
+      .returns("123")
+      .at_least_once
+
+    run_command("update", "-y").tap do |output|
+      assert_match "docker container stop traefik", output
+      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=traefik", output
+      assert_match "docker image prune --all --force --filter label=org.opencontainers.image.title=traefik", output
+      assert_match "docker container stop kamal-proxy", output
+      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy", output
+      assert_match "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}", output
+      assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target \"172.1.0.2:80\"", output
+    end
+  end
+
+  private
+    def run_command(*command)
+      stdouted { Kamal::Cli::Proxy.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
+    end
+end

test/cli/prune_test.rb

@@ -18,12 +18,10 @@ class CliPruneTest < CliTestCase
   test "containers" do
     run_command("containers").tap do |output|
       assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output
-      assert_match /docker container prune --force --filter label=service=healthcheck-app on 1.1.1.\d/, output
      end
 
     run_command("containers", "--retain", "10").tap do |output|
       assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +11 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output
-      assert_match /docker container prune --force --filter label=service=healthcheck-app on 1.1.1.\d/, output
     end
 
     assert_raises(RuntimeError, "retain must be at least 1") do

test/cli/server_test.rb

@@ -1,14 +1,30 @@
 require_relative "cli_test_case"
 
 class CliServerTest < CliTestCase
+  test "running a command with exec" do
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture)
+    .with("date", verbosity: 1)
+    .returns("Today")
+
+    hosts = "1.1.1.1".."1.1.1.4"
+    run_command("exec", "date").tap do |output|
+      hosts.map do |host|
+        assert_match "Running 'date' on #{hosts.to_a.join(', ')}...", output
+        assert_match "App Host: #{host}\nToday", output
+      end
+    end
+  end
+
   test "bootstrap already installed" do
+    stub_setup
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(true).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
 
-    assert_equal "", run_command("bootstrap")
+    assert_equal "Acquiring the deploy lock...\nReleasing the deploy lock...", run_command("bootstrap")
   end
 
   test "bootstrap install as non-root user" do
+    stub_setup
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', raise_on_non_zero_exit: false).returns(false).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
@@ -19,11 +35,13 @@ class CliServerTest < CliTestCase
   end
 
   test "bootstrap install as root user" do
+    stub_setup
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', raise_on_non_zero_exit: false).returns(true).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:sh, "-c", "'curl -fsSL https://get.docker.com || wget -O - https://get.docker.com || echo \"exit 1\"'", "|", :sh).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(".kamal/hooks/pre-connect", anything).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(".kamal/hooks/docker-setup", anything).at_least_once
 
     run_command("bootstrap").tap do |output|

test/cli/traefik_test.rb

@@ -1,96 +0,0 @@
-require_relative "cli_test_case"
-
-class CliTraefikTest < CliTestCase
-  test "boot" do
-    run_command("boot").tap do |output|
-      assert_match "docker login", output
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
-    end
-  end
-
-  test "reboot" do
-    Kamal::Commands::Registry.any_instance.expects(:login).twice
-
-    run_command("reboot", "-y").tap do |output|
-      assert_match "docker container stop traefik", output
-      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik", output
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
-    end
-  end
-
-  test "reboot --rolling" do
-    Object.any_instance.stubs(:sleep)
-
-    run_command("reboot", "--rolling", "-y").tap do |output|
-      assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.1", output
-    end
-  end
-
-  test "start" do
-    run_command("start").tap do |output|
-      assert_match "docker container start traefik", output
-    end
-  end
-
-  test "stop" do
-    run_command("stop").tap do |output|
-      assert_match "docker container stop traefik", output
-    end
-  end
-
-  test "restart" do
-    Kamal::Cli::Traefik.any_instance.expects(:stop)
-    Kamal::Cli::Traefik.any_instance.expects(:start)
-
-    run_command("restart")
-  end
-
-  test "details" do
-    run_command("details").tap do |output|
-      assert_match "docker ps --filter name=^traefik$", output
-    end
-  end
-
-  test "logs" do
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture)
-      .with(:docker, :logs, "traefik", " --tail 100", "--timestamps", "2>&1")
-      .returns("Log entry")
-
-    run_command("logs").tap do |output|
-      assert_match "Traefik Host: 1.1.1.1", output
-      assert_match "Log entry", output
-    end
-  end
-
-  test "logs with follow" do
-    SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 -p 22 'docker logs traefik --timestamps --tail 10 --follow 2>&1'")
-
-    assert_match "docker logs traefik --timestamps --tail 10 --follow", run_command("logs", "--follow")
-  end
-
-  test "remove" do
-    Kamal::Cli::Traefik.any_instance.expects(:stop)
-    Kamal::Cli::Traefik.any_instance.expects(:remove_container)
-    Kamal::Cli::Traefik.any_instance.expects(:remove_image)
-
-    run_command("remove")
-  end
-
-  test "remove_container" do
-    run_command("remove_container").tap do |output|
-      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik", output
-    end
-  end
-
-  test "remove_image" do
-    run_command("remove_image").tap do |output|
-      assert_match "docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik", output
-    end
-  end
-
-  private
-    def run_command(*command)
-      stdouted { Kamal::Cli::Traefik.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
-    end
-end

test/commander_test.rb

@@ -24,6 +24,9 @@ class CommanderTest < ActiveSupport::TestCase
     @kamal.specific_hosts = [ "*" ]
     assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @kamal.hosts
 
+    @kamal.specific_hosts = [ "1.1.1.[12]" ]
+    assert_equal [ "1.1.1.1", "1.1.1.2" ], @kamal.hosts
+
     exception = assert_raises(ArgumentError) do
       @kamal.specific_hosts = [ "*miss" ]
     end
@@ -57,6 +60,9 @@ class CommanderTest < ActiveSupport::TestCase
     @kamal.specific_roles = [ "*" ]
     assert_equal [ "web", "workers" ], @kamal.roles.map(&:name)
 
+    @kamal.specific_roles = [ "w{eb,orkers}" ]
+    assert_equal [ "web", "workers" ], @kamal.roles.map(&:name)
+
     exception = assert_raises(ArgumentError) do
       @kamal.specific_roles = [ "*miss" ]
     end
@@ -93,6 +99,11 @@ class CommanderTest < ActiveSupport::TestCase
     assert_equal [ "workers" ], @kamal.roles_on("1.1.1.3").map(&:name)
   end
 
+  test "roles_on web comes first" do
+    configure_with(:deploy_with_two_roles_one_host)
+    assert_equal [ "web", "workers" ], @kamal.roles_on("1.1.1.1").map(&:name)
+  end
+
   test "default group strategy" do
     assert_empty @kamal.boot_strategy
   end
@@ -119,9 +130,24 @@ class CommanderTest < ActiveSupport::TestCase
     configure_with(:deploy_primary_web_role_override)
 
     @kamal.specific_roles = [ "web_*" ]
-    assert_equal [ "web_chicago", "web_tokyo" ], @kamal.roles.map(&:name)
+    assert_equal [ "web_tokyo", "web_chicago" ], @kamal.roles.map(&:name)
     assert_equal "web_tokyo", @kamal.primary_role.name
     assert_equal "1.1.1.3", @kamal.primary_host
+    assert_equal [ "1.1.1.3", "1.1.1.4", "1.1.1.1", "1.1.1.2" ], @kamal.hosts
+  end
+
+  test "proxy hosts should observe filtered roles" do
+    configure_with(:deploy_with_aliases)
+
+    @kamal.specific_roles = [ "web_tokyo" ]
+    assert_equal [ "1.1.1.3", "1.1.1.4" ], @kamal.proxy_hosts
+  end
+
+  test "proxy hosts should observe filtered hosts" do
+    configure_with(:deploy_with_aliases)
+
+    @kamal.specific_hosts = [ "1.1.1.4" ]
+    assert_equal [ "1.1.1.4" ], @kamal.proxy_hosts
   end
 
   private

test/commands/app_test.rb

@@ -14,13 +14,13 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with hostname" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination dhh/app:999",
       new_command.run(hostname: "myhost").join(" ")
   end
 
@@ -28,31 +28,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:volumes] = [ "/local/path:/container/path" ]
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label destination dhh/app:999",
-      new_command.run.join(" ")
-  end
-
-  test "run with custom healthcheck path" do
-    @config[:healthcheck] = { "path" => "/healthz" }
-
-    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
-      new_command.run.join(" ")
-  end
-
-  test "run with custom healthcheck command" do
-    @config[:healthcheck] = { "cmd" => "/bin/up" }
-
-    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
-      new_command.run.join(" ")
-  end
-
-  test "run with role-specific healthcheck options" do
-    @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } }
-
-    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -60,14 +36,14 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:servers] = { "web" => [ "1.1.1.1" ], "jobs" => { "hosts" => [ "1.1.1.2" ], "cmd" => "bin/jobs", "options" => { "mount" => "somewhere", "cap-add" => true } } }
     assert_equal \
       "docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-jobs.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --label destination --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
-      new_command(role: "jobs").run.join(" ")
+      new_command(role: "jobs", host: "1.1.1.2").run.join(" ")
   end
 
   test "run with logging config" do
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label destination dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -76,7 +52,16 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "logging" => { "driver" => "local", "options" => { "max-size" => "100m" } } } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label destination dhh/app:999",
+      new_command.run.join(" ")
+  end
+
+  test "run with tags" do
+    @config[:servers] = [ { "1.1.1.1" => "tag1" } ]
+    @config[:env]["tags"] = { "tag1" => { "ENV1" => "value1" } }
+
+    assert_equal \
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -95,14 +80,14 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "stop" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop",
       new_command.stop.join(" ")
   end
 
   test "stop with custom stop wait time" do
     @config[:stop_wait_time] = 30
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop -t 30",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop -t 30",
       new_command.stop.join(" ")
   end
 
@@ -128,45 +113,45 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "logs" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs 2>&1",
       new_command.logs.join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --since 5m 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --since 5m 2>&1",
       new_command.logs(since: "5m").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --tail 100 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --tail 100 2>&1",
       new_command.logs(lines: "100").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --since 5m --tail 100 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --since 5m --tail 100 2>&1",
       new_command.logs(since: "5m", lines: "100").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs 2>&1 | grep 'my-id'",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs 2>&1 | grep 'my-id'",
       new_command.logs(grep: "my-id").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
       new_command.logs(since: "5m", grep: "my-id").join(" ")
   end
 
   test "follow logs" do
-    assert_match \
+    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --follow 2>&1",
+      "ssh -t root@app-1 -p 22 'sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --follow 2>&1'",
       new_command.follow_logs(host: "app-1")
 
-    assert_match \
+    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --follow 2>&1 | grep \"Completed\"",
+      "ssh -t root@app-1 -p 22 'sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --follow 2>&1 | grep \"Completed\"'",
       new_command.follow_logs(host: "app-1", grep: "Completed")
 
-    assert_match \
+    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 123 --follow 2>&1",
+      "ssh -t root@app-1 -p 22 'sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --tail 123 --follow 2>&1'",
       new_command.follow_logs(host: "app-1", lines: 123)
 
-    assert_match \
+    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 123 --follow 2>&1 | grep \"Completed\"",
+      "ssh -t root@app-1 -p 22 'sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --tail 123 --follow 2>&1 | grep \"Completed\"'",
       new_command.follow_logs(host: "app-1", lines: 123, grep: "Completed")
   end
 
@@ -174,36 +159,65 @@ class CommandsAppTest < ActiveSupport::TestCase
   test "execute in new container" do
     assert_equal \
       "docker run --rm --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails db:setup",
-      new_command.execute_in_new_container("bin/rails", "db:setup").join(" ")
+      new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ")
+  end
+
+  test "execute in new container with env" do
+    assert_equal \
+      "docker run --rm --env-file .kamal/env/roles/app-web.env --env foo=\"bar\" dhh/app:999 bin/rails db:setup",
+      new_command.execute_in_new_container("bin/rails", "db:setup", env: { "foo" => "bar" }).join(" ")
+  end
+
+  test "execute in new container with tags" do
+    @config[:servers] = [ { "1.1.1.1" => "tag1" } ]
+    @config[:env]["tags"] = { "tag1" => { "ENV1" => "value1" } }
+
+    assert_equal \
+      "docker run --rm --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" dhh/app:999 bin/rails db:setup",
+      new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ")
   end
 
   test "execute in new container with custom options" do
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } }
     assert_equal \
       "docker run --rm --env-file .kamal/env/roles/app-web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails db:setup",
-      new_command.execute_in_new_container("bin/rails", "db:setup").join(" ")
+      new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ")
   end
 
   test "execute in existing container" do
     assert_equal \
       "docker exec app-web-999 bin/rails db:setup",
-      new_command.execute_in_existing_container("bin/rails", "db:setup").join(" ")
+      new_command.execute_in_existing_container("bin/rails", "db:setup", env: {}).join(" ")
+  end
+
+  test "execute in existing container with env" do
+    assert_equal \
+      "docker exec --env foo=\"bar\" app-web-999 bin/rails db:setup",
+      new_command.execute_in_existing_container("bin/rails", "db:setup", env: { "foo" => "bar" }).join(" ")
   end
 
   test "execute in new container over ssh" do
     assert_match %r{docker run -it --rm --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails c},
-      new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1")
+      new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {})
+  end
+
+  test "execute in new container over ssh with tags" do
+    @config[:servers] = [ { "1.1.1.1" => "tag1" } ]
+    @config[:env]["tags"] = { "tag1" => { "ENV1" => "value1" } }
+
+    assert_equal "ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" dhh/app:999 bin/rails c'",
+      new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {})
   end
 
   test "execute in new container with custom options over ssh" do
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } }
     assert_match %r{docker run -it --rm --env-file .kamal/env/roles/app-web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails c},
-      new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1")
+      new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {})
   end
 
   test "execute in existing container over ssh" do
     assert_match %r{docker exec -it app-web-999 bin/rails c},
-      new_command.execute_in_existing_container_over_ssh("bin/rails", "c", host: "app-1")
+      new_command.execute_in_existing_container_over_ssh("bin/rails", "c", env: {})
   end
 
   test "run over ssh" do
@@ -242,14 +256,14 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "current_running_container_id" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest",
+    "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1",
       new_command.current_running_container_id.join(" ")
   end
 
   test "current_running_container_id with destination" do
     @destination = "staging"
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --filter status=restarting --latest",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest-staging --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --filter status=restarting' | head -1",
       new_command.current_running_container_id.join(" ")
   end
 
@@ -261,7 +275,7 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "current_running_version" do
     assert_equal \
-      "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done",
+      "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done",
       new_command.current_running_version.join(" ")
   end
 
@@ -339,10 +353,17 @@ class CommandsAppTest < ActiveSupport::TestCase
       new_command.remove_images.join(" ")
   end
 
-  test "tag_current_image_as_latest" do
+  test "tag_latest_image" do
     assert_equal \
       "docker tag dhh/app:999 dhh/app:latest",
-      new_command.tag_current_image_as_latest.join(" ")
+      new_command.tag_latest_image.join(" ")
+  end
+
+  test "tag_latest_image with destination" do
+    @destination = "staging"
+    assert_equal \
+      "docker tag dhh/app:999 dhh/app:latest-staging",
+      new_command.tag_latest_image.join(" ")
   end
 
   test "make_env_directory" do
@@ -353,25 +374,11 @@ class CommandsAppTest < ActiveSupport::TestCase
     assert_equal "rm -f .kamal/env/roles/app-web.env", new_command.remove_env_file.join(" ")
   end
 
-  test "cord" do
-    assert_equal "docker inspect -f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}' app-web-123 | awk '$2 == \"/tmp/kamal-cord\" {print $1}'", new_command.cord(version: 123).join(" ")
-  end
-
-  test "tie cord" do
-    assert_equal "mkdir -p . ; touch cordfile", new_command.tie_cord("cordfile").join(" ")
-    assert_equal "mkdir -p corddir ; touch corddir/cordfile", new_command.tie_cord("corddir/cordfile").join(" ")
-    assert_equal "mkdir -p /corddir ; touch /corddir/cordfile", new_command.tie_cord("/corddir/cordfile").join(" ")
-  end
-
-  test "cut cord" do
-    assert_equal "rm -r corddir", new_command.cut_cord("corddir").join(" ")
-  end
-
   test "extract assets" do
     assert_equal [
       :mkdir, "-p", ".kamal/assets/extracted/app-web-999", "&&",
       :docker, :stop, "-t 1", "app-web-assets", "2> /dev/null", "|| true", "&&",
-      :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "dhh/app:latest", "sleep 1000000", "&&",
+      :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "dhh/app:999", "sleep 1000000", "&&",
       :docker, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/assets/extracted/app-web-999", "&&",
       :docker, :stop, "-t 1", "app-web-assets"
     ], new_command(asset_path: "/public/assets").extract_assets
@@ -399,8 +406,8 @@ class CommandsAppTest < ActiveSupport::TestCase
   end
 
   private
-    def new_command(role: "web", **additional_config)
+    def new_command(role: "web", host: "1.1.1.1", **additional_config)
       config = Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999")
-      Kamal::Commands::App.new(config, role: config.role(role))
+      Kamal::Commands::App.new(config, role: config.role(role), host: host)
     end
 end

test/commands/builder_test.rb

@@ -9,7 +9,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "cache" => { "type" => "gha" } })
     assert_equal "multiarch", builder.name
     assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
       builder.push.join(" ")
   end
 
@@ -17,7 +17,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "multiarch" => false })
     assert_equal "native", builder.name
     assert_equal \
-      "git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
       builder.push.join(" ")
   end
 
@@ -25,7 +25,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "multiarch" => false, "cache" => { "type" => "gha" } })
     assert_equal "native/cached", builder.name
     assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
       builder.push.join(" ")
   end
 
@@ -33,7 +33,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "local" => {}, "remote" => {}, "cache" => { "type" => "gha" } })
     assert_equal "multiarch/remote", builder.name
     assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
       builder.push.join(" ")
   end
 
@@ -41,7 +41,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "local" => { "arch" => "amd64" } })
     assert_equal "multiarch", builder.name
     assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .",
       builder.push.join(" ")
   end
 
@@ -49,7 +49,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "remote" => { "arch" => "amd64" }, "cache" => { "type" => "gha" } })
     assert_equal "native/remote", builder.name
     assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64 --builder kamal-app-native-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64 --builder kamal-app-native-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
       builder.push.join(" ")
   end
 
@@ -83,6 +83,13 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     end
   end
 
+  test "build target" do
+    builder = new_builder_command(builder: { "target" => "prod" })
+    assert_equal \
+      "-t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --target prod",
+      builder.target.build_options.join(" ")
+  end
+
   test "build context" do
     builder = new_builder_command(builder: { "context" => ".." })
     assert_equal \
@@ -93,21 +100,21 @@ class CommandsBuilderTest < ActiveSupport::TestCase
   test "native push with build args" do
     builder = new_builder_command(builder: { "multiarch" => false, "args" => { "a" => 1, "b" => 2 } })
     assert_equal \
-      "git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
       builder.push.join(" ")
   end
 
   test "multiarch push with build args" do
     builder = new_builder_command(builder: { "args" => { "a" => 1, "b" => 2 } })
     assert_equal \
-      "git archive --format=tar HEAD | docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile -",
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile .",
       builder.push.join(" ")
   end
 
   test "native push with build secrets" do
     builder = new_builder_command(builder: { "multiarch" => false, "secrets" => [ "a", "b" ] })
     assert_equal \
-      "git archive --format=tar HEAD | docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile - && docker push dhh/app:123 && docker push dhh/app:latest",
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
       builder.push.join(" ")
   end
 
@@ -155,4 +162,8 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     def new_builder_command(additional_config = {})
       Kamal::Commands::Builder.new(Kamal::Configuration.new(@config.merge(additional_config), version: "123"))
     end
+
+    def build_directory
+      "#{Dir.tmpdir}/kamal-clones/app/kamal/"
+    end
 end

test/commands/healthcheck_test.rb

@@ -1,114 +0,0 @@
-require "test_helper"
-
-class CommandsHealthcheckTest < ActiveSupport::TestCase
-  setup do
-    @config = {
-      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
-      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
-    }
-  end
-
-  test "run" do
-    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "run with custom port" do
-    @config[:healthcheck] = { "port" => 3001 }
-
-    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3001 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3001/up || exit 1\" --health-interval \"1s\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "run with destination" do
-    @destination = "staging"
-
-    assert_equal \
-      "docker run --detach --name healthcheck-app-staging-123 --publish 3999:3000 --label service=healthcheck-app-staging -e KAMAL_CONTAINER_NAME=\"healthcheck-app-staging\" --env-file .kamal/env/roles/app-web-staging.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "run with custom healthcheck" do
-    @config[:healthcheck] = { "cmd" => "/bin/up" }
-
-    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/up\" --health-interval \"1s\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "run with custom options" do
-    @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere" } } }
-    @config[:healthcheck] = { "exposed_port" => 4999 }
-    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 4999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
-      new_command.run.join(" ")
-  end
-
-  test "status" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'",
-      new_command.status.join(" ")
-  end
-
-  test "container_health_log" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker inspect --format '{{json .State.Health}}'",
-      new_command.container_health_log.join(" ")
-  end
-
-  test "stop" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker stop",
-      new_command.stop.join(" ")
-  end
-
-  test "stop with destination" do
-    @destination = "staging"
-
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-staging-123$ --quiet | xargs docker stop",
-      new_command.stop.join(" ")
-  end
-
-  test "remove" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker container rm",
-      new_command.remove.join(" ")
-  end
-
-  test "remove with destination" do
-    @destination = "staging"
-
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-staging-123$ --quiet | xargs docker container rm",
-      new_command.remove.join(" ")
-  end
-
-  test "logs" do
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker logs --tail 50 2>&1",
-      new_command.logs.join(" ")
-  end
-
-  test "logs with custom lines number" do
-    @config[:healthcheck] = { "log_lines" => 150 }
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker logs --tail 150 2>&1",
-      new_command.logs.join(" ")
-  end
-
-  test "logs with destination" do
-    @destination = "staging"
-
-    assert_equal \
-      "docker container ls --all --filter name=^healthcheck-app-staging-123$ --quiet | xargs docker logs --tail 50 2>&1",
-      new_command.logs.join(" ")
-  end
-
-  private
-    def new_command
-      Kamal::Commands::Healthcheck.new(Kamal::Configuration.new(@config, destination: @destination, version: "123"))
-    end
-end

test/commands/hook_test.rb

@@ -7,8 +7,7 @@ class CommandsHookTest < ActiveSupport::TestCase
     freeze_time
 
     @config = {
-      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ]
-      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
     }
 
     @performer = `whoami`.strip

test/commands/lock_test.rb

@@ -3,8 +3,7 @@ require "test_helper"
 class CommandsLockTest < ActiveSupport::TestCase
   setup do
     @config = {
-      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ]
-      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
     }
   end
 

test/commands/proxy_test.rb

@@ -0,0 +1,126 @@
+require "test_helper"
+
+class CommandsProxyTest < ActiveSupport::TestCase
+  setup do
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ]
+    }
+
+    ENV["EXAMPLE_API_KEY"] = "456"
+  end
+
+  teardown do
+    ENV.delete("EXAMPLE_API_KEY")
+  end
+
+  test "run" do
+    assert_equal \
+      "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}",
+      new_command.run.join(" ")
+  end
+
+  test "run with ports configured" do
+    assert_equal \
+      "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}",
+      new_command.run.join(" ")
+  end
+
+  test "run without configuration" do
+    @config.delete(:proxy)
+
+    assert_equal \
+      "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}",
+      new_command.run.join(" ")
+  end
+
+  test "run with logging config" do
+    @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
+
+    assert_equal \
+      "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}",
+      new_command.run.join(" ")
+  end
+
+  test "proxy start" do
+    assert_equal \
+      "docker container start kamal-proxy",
+      new_command.start.join(" ")
+  end
+
+  test "proxy stop" do
+    assert_equal \
+      "docker container stop kamal-proxy",
+      new_command.stop.join(" ")
+  end
+
+  test "proxy info" do
+    assert_equal \
+      "docker ps --filter name=^kamal-proxy$",
+      new_command.info.join(" ")
+  end
+
+  test "proxy logs" do
+    assert_equal \
+      "docker logs kamal-proxy --timestamps 2>&1",
+      new_command.logs.join(" ")
+  end
+
+  test "proxy logs since 2h" do
+    assert_equal \
+      "docker logs kamal-proxy  --since 2h --timestamps 2>&1",
+      new_command.logs(since: "2h").join(" ")
+  end
+
+  test "proxy logs last 10 lines" do
+    assert_equal \
+      "docker logs kamal-proxy  --tail 10 --timestamps 2>&1",
+      new_command.logs(lines: 10).join(" ")
+  end
+
+  test "proxy logs with grep hello!" do
+    assert_equal \
+      "docker logs kamal-proxy --timestamps 2>&1 | grep 'hello!'",
+      new_command.logs(grep: "hello!").join(" ")
+  end
+
+  test "proxy remove container" do
+    assert_equal \
+      "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy",
+      new_command.remove_container.join(" ")
+  end
+
+  test "proxy remove image" do
+    assert_equal \
+      "docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy",
+      new_command.remove_image.join(" ")
+  end
+
+  test "proxy follow logs" do
+    assert_equal \
+      "ssh -t root@1.1.1.1 -p 22 'docker logs kamal-proxy --timestamps --tail 10 --follow 2>&1'",
+      new_command.follow_logs(host: @config[:servers].first)
+  end
+
+  test "proxy follow logs with grep hello!" do
+    assert_equal \
+      "ssh -t root@1.1.1.1 -p 22 'docker logs kamal-proxy --timestamps --tail 10 --follow 2>&1 | grep \"hello!\"'",
+      new_command.follow_logs(host: @config[:servers].first, grep: "hello!")
+  end
+
+  test "deploy" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy deploy service --target \"172.1.0.2:80\"",
+      new_command.deploy("service", target: "172.1.0.2:80").join(" ")
+  end
+
+  test "remove" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy remove service --target \"172.1.0.2:80\"",
+      new_command.remove("service", target: "172.1.0.2:80").join(" ")
+  end
+
+  private
+    def new_command
+      Kamal::Commands::Proxy.new(Kamal::Configuration.new(@config, version: "123"))
+    end
+end

test/commands/prune_test.rb

@@ -3,8 +3,7 @@ require "test_helper"
 class CommandsPruneTest < ActiveSupport::TestCase
   setup do
     @config = {
-      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ]
-      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
     }
   end
 
@@ -30,12 +29,6 @@ class CommandsPruneTest < ActiveSupport::TestCase
       new_command.app_containers(retain: 3).join(" ")
   end
 
-  test "healthcheck containers" do
-    assert_equal \
-      "docker container prune --force --filter label=service=healthcheck-app",
-      new_command.healthcheck_containers.join(" ")
-  end
-
   private
     def new_command
       Kamal::Commands::Prune.new(Kamal::Configuration.new(@config, version: "123"))

test/commands/server_test.rb

@@ -3,8 +3,7 @@ require "test_helper"
 class CommandsServerTest < ActiveSupport::TestCase
   setup do
     @config = {
-      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ]
-      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
     }
   end
 

test/commands/traefik_test.rb

@@ -1,198 +0,0 @@
-require "test_helper"
-
-class CommandsTraefikTest < ActiveSupport::TestCase
-  setup do
-    @image = "traefik:test"
-
-    @config = {
-      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
-      traefik: { "image" => @image, "args" => { "accesslog.format" => "json", "api.insecure" => true, "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
-    }
-
-    ENV["EXAMPLE_API_KEY"] = "456"
-  end
-
-  teardown do
-    ENV.delete("EXAMPLE_API_KEY")
-  end
-
-  test "run" do
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-
-    @config[:traefik]["host_port"] = "8080"
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-
-    @config[:traefik]["publish"] = false
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-  end
-
-  test "run with ports configured" do
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-
-    @config[:traefik]["options"] = { "publish" => %w[9000:9000 9001:9001] }
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-  end
-
-  test "run with volumes configured" do
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-
-    @config[:traefik]["options"] = { "volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json] }
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-  end
-
-  test "run with several options configured" do
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-
-    @config[:traefik]["options"] = { "volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json], "publish" => %w[8080:8080], "memory" => "512m" }
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-  end
-
-  test "run with labels configured" do
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-
-    @config[:traefik]["labels"] = { "traefik.http.routers.dashboard.service" => "api@internal", "traefik.http.routers.dashboard.middlewares" => "auth" }
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" --label traefik.http.routers.dashboard.service=\"api@internal\" --label traefik.http.routers.dashboard.middlewares=\"auth\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-  end
-
-  test "run with env configured" do
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-
-    @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-  end
-
-  test "run without configuration" do
-    @config.delete(:traefik)
-
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"",
-      new_command.run.join(" ")
-  end
-
-  test "run with logging config" do
-    @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
-
-    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-  end
-
-  test "run with default args overriden" do
-    @config[:traefik]["args"]["log.level"] = "ERROR"
-
-    assert_equal \
-    "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"ERROR\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
-      new_command.run.join(" ")
-  end
-
-  test "traefik start" do
-    assert_equal \
-      "docker container start traefik",
-      new_command.start.join(" ")
-  end
-
-  test "traefik stop" do
-    assert_equal \
-      "docker container stop traefik",
-      new_command.stop.join(" ")
-  end
-
-  test "traefik info" do
-    assert_equal \
-      "docker ps --filter name=^traefik$",
-      new_command.info.join(" ")
-  end
-
-  test "traefik logs" do
-    assert_equal \
-      "docker logs traefik --timestamps 2>&1",
-      new_command.logs.join(" ")
-  end
-
-  test "traefik logs since 2h" do
-    assert_equal \
-      "docker logs traefik  --since 2h --timestamps 2>&1",
-      new_command.logs(since: "2h").join(" ")
-  end
-
-  test "traefik logs last 10 lines" do
-    assert_equal \
-      "docker logs traefik  --tail 10 --timestamps 2>&1",
-      new_command.logs(lines: 10).join(" ")
-  end
-
-  test "traefik logs with grep hello!" do
-    assert_equal \
-      "docker logs traefik --timestamps 2>&1 | grep 'hello!'",
-      new_command.logs(grep: "hello!").join(" ")
-  end
-
-  test "traefik remove container" do
-    assert_equal \
-      "docker container prune --force --filter label=org.opencontainers.image.title=Traefik",
-      new_command.remove_container.join(" ")
-  end
-
-  test "traefik remove image" do
-    assert_equal \
-      "docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik",
-      new_command.remove_image.join(" ")
-  end
-
-  test "traefik follow logs" do
-    assert_equal \
-      "ssh -t root@1.1.1.1 -p 22 'docker logs traefik --timestamps --tail 10 --follow 2>&1'",
-      new_command.follow_logs(host: @config[:servers].first)
-  end
-
-  test "traefik follow logs with grep hello!" do
-    assert_equal \
-      "ssh -t root@1.1.1.1 -p 22 'docker logs traefik --timestamps --tail 10 --follow 2>&1 | grep \"hello!\"'",
-      new_command.follow_logs(host: @config[:servers].first, grep: "hello!")
-  end
-
-  test "secrets io" do
-    @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
-
-    assert_equal "EXAMPLE_API_KEY=456\n", new_command.env.secrets_io.string
-  end
-
-  test "make_env_directory" do
-    assert_equal "mkdir -p .kamal/env/traefik", new_command.make_env_directory.join(" ")
-  end
-
-  test "remove_env_file" do
-    assert_equal "rm -f .kamal/env/traefik/traefik.env", new_command.remove_env_file.join(" ")
-  end
-
-  private
-    def new_command
-      Kamal::Commands::Traefik.new(Kamal::Configuration.new(@config, version: "123"))
-    end
-end

test/configuration/accessory_test.rb

@@ -107,6 +107,10 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
     assert_equal "Specify one of `host`, `hosts` or `roles` for accessory `mysql`", exception.message
   end
 
+  test "all hosts" do
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4", "1.1.1.5", "1.1.1.6", "1.1.1.7" ], @config.all_hosts
+  end
+
   test "label args" do
     assert_equal [ "--label", "service=\"app-mysql\"" ], @config.accessory(:mysql).label_args
     assert_equal [ "--label", "service=\"app-redis\"", "--label", "cache=\"true\"" ], @config.accessory(:redis).label_args

test/configuration/builder_test.rb

@@ -140,7 +140,7 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase
   end
 
   test "context" do
-    assert_equal "-", @config.builder.context
+    assert_equal ".", @config.builder.context
   end
 
   test "setting context" do

test/configuration/env/tags_test.rb

@@ -0,0 +1,112 @@
+require "test_helper"
+
+class ConfigurationEnvTagsTest < ActiveSupport::TestCase
+  setup do
+    @deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+      servers: [ { "1.1.1.1" => "odd" }, { "1.1.1.2" => "even" }, { "1.1.1.3" => [ "odd", "three" ] } ],
+      env: {
+        "clear" => { "REDIS_URL" => "redis://x/y", "THREE" => "false" },
+        "tags" => {
+          "odd" => { "TYPE" => "odd" },
+          "even" => { "TYPE" => "even" },
+          "three" => { "THREE" => "true" }
+        }
+      }
+    }
+
+    @config = Kamal::Configuration.new(@deploy)
+
+    @deploy_with_roles = @deploy.dup.merge({
+      servers: {
+        "web" => [ { "1.1.1.1" => "odd" }, "1.1.1.2" ],
+        "workers" => {
+          "hosts" => [ { "1.1.1.3" => [ "odd", "oddjob" ] }, "1.1.1.4" ],
+          "cmd" => "bin/jobs",
+          "env" => {
+            "REDIS_URL" => "redis://a/b",
+            "WEB_CONCURRENCY" => 4
+          }
+        }
+      },
+      env: {
+        "tags" => {
+          "odd" => { "TYPE" => "odd" },
+          "oddjob" => { "TYPE" => "oddjob" }
+        }
+      }
+    })
+
+    @config_with_roles = Kamal::Configuration.new(@deploy_with_roles)
+  end
+
+  test "tags" do
+    assert_equal 3, @config.env_tags.size
+    assert_equal %w[ odd even three ], @config.env_tags.map(&:name)
+    assert_equal({ "TYPE" => "odd" }, @config.env_tag("odd").env.clear)
+    assert_equal({ "TYPE" => "even" }, @config.env_tag("even").env.clear)
+    assert_equal({ "THREE" => "true" }, @config.env_tag("three").env.clear)
+  end
+
+  test "tags with roles" do
+    assert_equal 2, @config_with_roles.env_tags.size
+    assert_equal %w[ odd oddjob ], @config_with_roles.env_tags.map(&:name)
+    assert_equal({ "TYPE" => "odd" }, @config_with_roles.env_tag("odd").env.clear)
+    assert_equal({ "TYPE" => "oddjob" }, @config_with_roles.env_tag("oddjob").env.clear)
+  end
+
+  test "tag overrides env" do
+    assert_equal "false", @config.role("web").env("1.1.1.1").clear["THREE"]
+    assert_equal "true", @config.role("web").env("1.1.1.3").clear["THREE"]
+  end
+
+  test "later tag wins" do
+    deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+      servers: [ { "1.1.1.1" => [ "first", "second" ] } ],
+      env: {
+        "tags" => {
+          "first" => { "TYPE" => "first" },
+          "second" => { "TYPE" => "second" }
+        }
+      }
+    }
+
+    config = Kamal::Configuration.new(deploy)
+    assert_equal "second", config.role("web").env("1.1.1.1").clear["TYPE"]
+  end
+
+  test "tag secret env" do
+    ENV["PASSWORD"] = "hello"
+
+    deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+      servers: [ { "1.1.1.1" => "secrets" } ],
+      env: {
+        "tags" => {
+          "secrets" => { "secret" => [ "PASSWORD" ] }
+        }
+      }
+    }
+
+    config = Kamal::Configuration.new(deploy)
+    assert_equal "hello", config.role("web").env("1.1.1.1").secrets["PASSWORD"]
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "tag clear env" do
+    deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+      servers: [ { "1.1.1.1" => "clearly" } ],
+      env: {
+        "tags" => {
+          "clearly" => { "clear" => { "FOO" => "bar" } }
+        }
+      }
+    }
+
+    config = Kamal::Configuration.new(deploy)
+    assert_equal "bar", config.role("web").env("1.1.1.1").clear["FOO"]
+  end
+end

test/configuration/role_test.rb

@@ -42,7 +42,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
   end
 
   test "special label args for web" do
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "destination", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.routers.app-web.priority=\"2\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "destination" ], @config.role(:web).label_args
   end
 
   test "custom labels" do
@@ -56,24 +56,11 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     assert_equal "70", @config_with_roles.role(:workers).labels["my.custom.label"]
   end
 
-  test "overwriting default traefik label" do
-    @deploy[:labels] = { "traefik.http.routers.app-web.rule" => "\"Host(\\`example.com\\`) || (Host(\\`example.org\\`) && Path(\\`/traefik\\`))\"" }
-    assert_equal "\"Host(\\`example.com\\`) || (Host(\\`example.org\\`) && Path(\\`/traefik\\`))\"", @config.role(:web).labels["traefik.http.routers.app-web.rule"]
-  end
-
-  test "default traefik label on non-web role" do
-    config = Kamal::Configuration.new(@deploy_with_roles.tap { |c|
-      c[:servers]["beta"] = { "traefik" => "true", "hosts" => [ "1.1.1.5" ] }
-    })
-
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "destination", "--label", "traefik.http.services.app-beta.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.routers.app-beta.priority=\"2\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args
-  end
-
   test "env overwritten by role" do
-    assert_equal "redis://a/b", @config_with_roles.role(:workers).env.clear["REDIS_URL"]
+    assert_equal "redis://a/b", @config_with_roles.role(:workers).env("1.1.1.3").clear["REDIS_URL"]
 
-    assert_equal "\n", @config_with_roles.role(:workers).env.secrets_io.string
+    assert_equal "\n", @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
-    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
   end
 
   test "container name" do
@@ -86,7 +73,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
   end
 
   test "env args" do
-    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
   end
 
   test "env secret overwritten by role" do
@@ -117,8 +104,8 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       DB_PASSWORD=secret&\"123
     ENV
 
-    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env.secrets_io.string
+    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
-    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
   ensure
     ENV["REDIS_PASSWORD"] = nil
     ENV["DB_PASSWORD"] = nil
@@ -141,8 +128,8 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       DB_PASSWORD=secret123
     ENV
 
-    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env.secrets_io.string
+    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
-    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
   ensure
     ENV["DB_PASSWORD"] = nil
   end
@@ -163,8 +150,8 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       REDIS_PASSWORD=secret456
     ENV
 
-    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env.secrets_io.string
+    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
-    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
   ensure
     ENV["REDIS_PASSWORD"] = nil
   end
@@ -191,34 +178,14 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       REDIS_PASSWORD=secret456
     ENV
 
-    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env.secrets_io.string
+    assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
-    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://c/d\"" ], @config_with_roles.role(:workers).env_args
+    assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://c/d\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
   ensure
     ENV["REDIS_PASSWORD"] = nil
   end
 
   test "env secrets_file" do
-    assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).env.secrets_file
+    assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).env("1.1.1.3").secrets_file
-  end
-
-  test "uses cord" do
-    assert @config_with_roles.role(:web).uses_cord?
-    assert_not @config_with_roles.role(:workers).uses_cord?
-  end
-
-  test "cord host file" do
-    assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}/cord}, @config_with_roles.role(:web).cord_host_file
-  end
-
-  test "cord volume" do
-    assert_equal "/tmp/kamal-cord", @config_with_roles.role(:web).cord_volume.container_path
-    assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}}, @config_with_roles.role(:web).cord_volume.host_path
-    assert_equal "--volume", @config_with_roles.role(:web).cord_volume.docker_args[0]
-    assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}:/tmp/kamal-cord}, @config_with_roles.role(:web).cord_volume.docker_args[1]
-  end
-
-  test "cord container file" do
-    assert_equal "/tmp/kamal-cord/cord", @config_with_roles.role(:web).cord_container_file
   end
 
   test "asset path and volume args" do

test/configuration_test.rb

@@ -44,6 +44,7 @@ class ConfigurationTest < ActiveSupport::TestCase
 
   test "service name valid" do
     assert Kamal::Configuration.new(@deploy.tap { _1[:service] = "hey-app1_primary" }).valid?
+    assert Kamal::Configuration.new(@deploy.tap { _1[:service] = "MyApp" }).valid?
   end
 
   test "service name invalid" do
@@ -73,13 +74,22 @@ class ConfigurationTest < ActiveSupport::TestCase
     assert_equal "1.1.1.1", @config_with_roles.primary_host
   end
 
-  test "traefik hosts" do
+  test "proxy hosts" do
-    assert_equal [ "1.1.1.1", "1.1.1.2" ], @config_with_roles.traefik_hosts
+    assert_equal [ "1.1.1.1", "1.1.1.2" ], @config_with_roles.proxy_hosts
 
-    @deploy_with_roles[:servers]["workers"]["traefik"] = true
+    @deploy_with_roles[:servers]["workers"]["proxy"] = true
     config = Kamal::Configuration.new(@deploy_with_roles)
 
-    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], config.traefik_hosts
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], config.proxy_hosts
+  end
+
+  test "filtered proxy hosts" do
+    assert_equal [ "1.1.1.1", "1.1.1.2" ], @config_with_roles.proxy_hosts
+
+    @deploy_with_roles[:servers]["workers"]["proxy"] = true
+    config = Kamal::Configuration.new(@deploy_with_roles)
+
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], config.proxy_hosts
   end
 
   test "version no git repo" do
@@ -144,10 +154,6 @@ class ConfigurationTest < ActiveSupport::TestCase
     assert_equal "app-missing", @config.service_with_version
   end
 
-  test "healthcheck service" do
-    assert_equal "healthcheck-app", @config.healthcheck_service
-  end
-
   test "valid config" do
     assert @config.valid?
     assert @config_with_roles.valid?
@@ -261,8 +267,7 @@ class ConfigurationTest < ActiveSupport::TestCase
         sshkit: {},
         volume_args: [ "--volume", "/local/path:/container/path" ],
         builder: {},
-        logging: [ "--log-opt", "max-size=\"10m\"" ],
+        logging: [ "--log-opt", "max-size=\"10m\"" ] }
-        healthcheck: { "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 } }
 
     assert_equal expected_config, @config.to_h
   end
@@ -320,7 +325,7 @@ class ConfigurationTest < ActiveSupport::TestCase
     assert_equal "alternate_web", config.primary_role.name
     assert_equal "1.1.1.4", config.primary_host
     assert config.role(:alternate_web).primary?
-    assert config.role(:alternate_web).running_traefik?
+    assert config.role(:alternate_web).running_proxy?
   end
 
   test "primary role missing" do

test/env_file_test.rb

@@ -11,6 +11,33 @@ class EnvFileTest < ActiveSupport::TestCase
       Kamal::EnvFile.new(env).to_s
   end
 
+  test "to_str won't escape chinese characters" do
+    env = {
+      "foo" => '你好 means hello, "欢迎" means welcome, that\'s simple! 😃 {smile}'
+    }
+
+    assert_equal "foo=你好 means hello, \"欢迎\" means welcome, that's simple! 😃 {smile}\n",
+      Kamal::EnvFile.new(env).to_s
+  end
+
+  test "to_s won't escape japanese characters" do
+    env = {
+      "foo" => 'こんにちは means hello, "ようこそ" means welcome, that\'s simple! 😃 {smile}'
+    }
+
+    assert_equal "foo=こんにちは means hello, \"ようこそ\" means welcome, that's simple! 😃 {smile}\n", \
+      Kamal::EnvFile.new(env).to_s
+  end
+
+  test "to_s won't escape korean characters" do
+    env = {
+      "foo" => '안녕하세요 means hello, "어서 오십시오" means welcome, that\'s simple! 😃 {smile}'
+    }
+
+    assert_equal "foo=안녕하세요 means hello, \"어서 오십시오\" means welcome, that's simple! 😃 {smile}\n", \
+      Kamal::EnvFile.new(env).to_s
+  end
+
   test "to_s empty" do
     assert_equal "\n", Kamal::EnvFile.new({}).to_s
   end

test/fixtures/deploy_primary_web_role_override.yml

@@ -2,12 +2,12 @@ service: app
 image: dhh/app
 servers:
   web_chicago:
-    traefik: enabled
+    proxy: enabled
     hosts:
       - 1.1.1.1
       - 1.1.1.2
   web_tokyo:
-    traefik: enabled
+    proxy: enabled
     hosts:
       - 1.1.1.3
       - 1.1.1.4

test/fixtures/deploy_with_aliases.yml

@@ -10,7 +10,7 @@ tokyo_hosts: &tokyo_hosts
 web_common: &web_common
   env:
     ROLE: "web"
-  traefik: true
+  proxy: true
 
 # actual config
 service: app

test/fixtures/deploy_with_env_tags.yml

@@ -0,0 +1,28 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - 1.1.1.1: site1
+    - 1.1.1.2: [ site1 experimental ]
+    - 1.2.1.1: site2
+    - 1.2.1.2: site2
+  workers:
+    - 1.1.1.3: site1
+    - 1.1.1.4: site1
+    - 1.2.1.3: site2
+    - 1.2.1.4: [ site2 experimental ]
+env:
+  clear:
+    TEST: "root"
+    EXPERIMENT: "disabled"
+  tags:
+    site1:
+      SITE: site1
+    site2:
+      SITE: site2
+    experimental:
+      EXPERIMENT: "enabled"
+
+registry:
+  username: user
+  password: pw

test/fixtures/deploy_with_remote_builder_and_custom_ports.yml

@@ -0,0 +1,45 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+registry:
+  username: user
+  password: pw
+
+accessories:
+  mysql:
+    image: mysql:5.7
+    host: 1.1.1.3
+    port: 3306
+    env:
+      clear:
+        MYSQL_ROOT_HOST: '%'
+      secret:
+        - MYSQL_ROOT_PASSWORD
+    files:
+      - test/fixtures/files/my.cnf:/etc/mysql/my.cnf
+    directories:
+      - data:/var/lib/mysql
+  redis:
+    image: redis:latest
+    roles:
+      - web
+    port: 6379
+    directories:
+      - data:/data
+
+readiness_delay: 0
+
+ssh:
+  user: root
+  port: 22
+
+builder:
+  remote:
+    arch: amd64
+    host: ssh://app@1.1.1.5:2122

test/fixtures/deploy_with_two_roles_one_host.yml

@@ -0,0 +1,15 @@
+service: app
+image: dhh/app
+servers:
+  workers:
+    hosts:
+      - 1.1.1.1
+  web:
+    hosts:
+      - 1.1.1.1
+env:
+  REDIS_URL: redis://x/y
+registry:
+  server: registry.digitalocean.com
+  username: user
+  password: pw

test/fixtures/deploy_with_uncommon_hostnames.yml

@@ -0,0 +1,8 @@
+service: app
+image: dhh/app
+servers:
+  - "this-hostname-with-random-part-is-too-long.example.com"
+  - "this-hostname-is-really-unacceptably-long-to-be-honest.example.com"
+registry:
+  username: user
+  password: pw

test/fixtures/deploy_without_clone.yml

@@ -0,0 +1,39 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+registry:
+  username: user
+  password: pw
+
+accessories:
+  mysql:
+    image: mysql:5.7
+    host: 1.1.1.3
+    port: 3306
+    env:
+      clear:
+        MYSQL_ROOT_HOST: '%'
+      secret:
+        - MYSQL_ROOT_PASSWORD
+    files:
+      - test/fixtures/files/my.cnf:/etc/mysql/my.cnf
+    directories:
+      - data:/var/lib/mysql
+  redis:
+    image: redis:latest
+    roles:
+      - web
+    port: 6379
+    directories:
+      - data:/data
+
+readiness_delay: 0
+
+builder:
+  context: "."

test/fixtures/deploy_workers_only.yml

@@ -2,7 +2,7 @@ service: app
 image: dhh/app
 servers:
   workers:
-    traefik: false
+    proxy: false
     hosts:
       - 1.1.1.1
       - 1.1.1.2

test/integration/accessory_test.rb

@@ -1,6 +1,6 @@
 require_relative "integration_test"
 
-class AccessoryTest < IntegrationTest
+class IntegrationAccessoryTest < IntegrationTest
   test "boot, stop, start, restart, logs, remove" do
     kamal :envify
 

test/integration/app_test.rb

@@ -1,16 +1,18 @@
 require_relative "integration_test"
 
-class AppTest < IntegrationTest
+class IntegrationAppTest < IntegrationTest
   test "stop, start, boot, logs, images, containers, exec, remove" do
     kamal :envify
 
+    kamal :setup
+
     kamal :deploy
 
     assert_app_is_up
 
     kamal :app, :stop
 
-    assert_app_is_down
+    assert_app_is_down response_code: "504"
 
     kamal :app, :start
 
@@ -24,7 +26,7 @@ class AppTest < IntegrationTest
     logs = kamal :app, :logs, capture: true
     assert_match /App Host: vm1/, logs
     assert_match /App Host: vm2/, logs
-    assert_match /GET \/ HTTP\/1.1/, logs
+    assert_match /GET \/up HTTP\/1.1/, logs
 
     images = kamal :app, :images, capture: true
     assert_match /App Host: vm1/, images
@@ -50,6 +52,6 @@ class AppTest < IntegrationTest
 
     kamal :app, :remove
 
-    assert_app_is_down
+    assert_app_is_down response_code: "504"
   end
 end

test/integration/broken_deploy_test.rb

@@ -0,0 +1,33 @@
+require_relative "integration_test"
+
+class BrokenDeployTest < IntegrationTest
+  test "deploying a bad image" do
+    @app = "app_with_roles"
+
+    kamal :envify
+
+    first_version = latest_app_version
+
+    kamal :deploy
+
+    assert_app_is_up version: first_version
+    assert_container_running host: :vm3, name: "app-workers-#{first_version}"
+
+    second_version = break_app
+
+    output = kamal :deploy, raise_on_error: false, capture: true
+
+    assert_failed_deploy output
+    assert_app_is_up version: first_version
+    assert_container_running host: :vm3, name: "app-workers-#{first_version}"
+    assert_container_not_running host: :vm3, name: "app-workers-#{second_version}"
+  end
+
+  private
+    def assert_failed_deploy(output)
+      assert_match "Waiting for the first healthy web container before booting workers on vm3...", output
+      assert_match /First web container is unhealthy on vm[12], not booting other roles/, output
+      assert_match "First web container is unhealthy, not booting workers on vm3", output
+      assert_match "nginx: [emerg] unexpected end of file, expecting \";\" or \"}\" in /etc/nginx/conf.d/default.conf:2", output
+    end
+end