WIP

Zero downtime deployment with cord file

bin/kamal

@@ -9,10 +9,10 @@ begin
   Kamal::Cli::Main.start(ARGV)
 rescue SSHKit::Runner::ExecuteError => e
   puts "  \e[31mERROR (#{e.cause.class}): #{e.message}\e[0m"
-  puts e.cause.backtrace
+  puts e.cause.backtrace if ENV["VERBOSE"]
   exit 1
 rescue => e
   puts "  \e[31mERROR (#{e.class}): #{e.message}\e[0m"
-  puts e.backtrace
+  puts e.backtrace if ENV["VERBOSE"]
   exit 1
 end

lib/kamal/cli.rb

@@ -1,7 +1,6 @@
 module Kamal::Cli
   class LockError < StandardError; end
   class HookError < StandardError; end
-  class TraefikError < StandardError; end
 end
 
 # SSHKit uses instance eval, so we need a global const for ergonomics

lib/kamal/cli/app.rb

@@ -2,8 +2,6 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "boot", "Boot app on servers (or reboot app if already running)"
   def boot
     mutating do
-      ensure_traefik_file_provider_enabled
-
       hold_lock_on_error do
         say "Get most recent version available as an image...", :magenta unless options[:version]
         using_version(version_or_latest) do |version|
@@ -20,30 +18,44 @@ class Kamal::Cli::App < Kamal::Cli::Base
             roles.each do |role|
               app = KAMAL.app(role: role)
               auditor = KAMAL.auditor(role: role)
-              traefik_dynamic = KAMAL.traefik_dynamic(role: role)
               role_config = KAMAL.config.role(role)
 
-              if capture_with_info(*app.container_id_for_version(version, only_running: true), raise_on_non_zero_exit: false).present?
+              execute *app.extract_assets if role_config.assets?
+
+
+              if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
                 tmp_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
                 info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
                 execute *auditor.record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
                 execute *app.rename_container(version: version, new_version: tmp_version)
               end
 
+              old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+              original_old_version = old_version.gsub(/_replaced_[a-f0-9]{16}$/, "")
+
+              execute *app.sync_asset_volumes(old_version: original_old_version) if role_config.assets?
+
+              execute *app.tie_cord(role_config.cord_host_file) if role_config.uses_cord?
+
               execute *auditor.record("Booted app version #{version}"), verbosity: :debug
 
-              old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+              execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
-              execute *app.start_or_run(hostname: "#{host}-#{SecureRandom.hex(6)}")
 
               Kamal::Utils::HealthcheckPoller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
 
-              if role_config.running_traefik?
+              if old_version.present?
-                ip_address = capture_with_info(*app.ip_address(version: version)).strip
+                if role_config.uses_cord?
-                execute *traefik_dynamic.write_config(ip_address: ip_address)
+                  cord = capture_with_info(*app.cord(version: old_version), raise_on_non_zero_exit: false).strip
-                Kamal::Utils::SwitchPoller.wait_for_switch(traefik_dynamic) { capture_with_info(*traefik_dynamic.run_id)&.strip }
+                  if cord.present?
-              end
+                    execute *app.cut_cord(cord)
+                    Kamal::Utils::HealthcheckPoller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
+                  end
+                end
 
-              execute *app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
+                execute *app.stop(version: old_version), raise_on_non_zero_exit: false
+
+                execute *app.cleanup_assets if role_config.assets?
+              end
             end
           end
         end
@@ -54,23 +66,12 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "start", "Start existing app container on servers"
   def start
     mutating do
-      ensure_traefik_file_provider_enabled
-
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
-          app = KAMAL.app(role: role)
-          role_config = KAMAL.config.role(role)
-
           execute *KAMAL.auditor.record("Started app version #{KAMAL.config.version}"), verbosity: :debug
-          execute *app.start, raise_on_non_zero_exit: false
+          execute *KAMAL.app(role: role).start, raise_on_non_zero_exit: false
-          version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-
-          if role_config.running_traefik?
-            ip_address = capture_with_info(*app.ip_address(version: version)).strip
-            execute *KAMAL.traefik_dynamic(role: role).write_config(ip_address: ip_address)
-          end
         end
       end
     end
@@ -83,10 +84,8 @@ class Kamal::Cli::App < Kamal::Cli::Base
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
-          app = KAMAL.app(role: role)
           execute *KAMAL.auditor.record("Stopped app", role: role), verbosity: :debug
-          execute *KAMAL.traefik_dynamic(role: role).remove_config if KAMAL.config.role(role).running_traefik?
+          execute *KAMAL.app(role: role).stop, raise_on_non_zero_exit: false
-          execute *app.stop, raise_on_non_zero_exit: false
         end
       end
     end
@@ -316,13 +315,4 @@ class Kamal::Cli::App < Kamal::Cli::Base
     def version_or_latest
       options[:version] || "latest"
     end
-
-    def ensure_traefik_file_provider_enabled
-      # Ensure traefik has been rebooted to switch to the file provider
-      on(KAMAL.traefik_hosts) do
-        unless capture_with_info(*KAMAL.traefik_static.docker_entrypoint_args).include?("--providers.file.directory=")
-          raise Kamal::Cli::TraefikError, "File provider not enabled, you'll need to run `kamal traefik reboot` to deploy"
-        end
-      end
-    end
 end

lib/kamal/cli/env.rb

@@ -13,9 +13,8 @@ class Kamal::Cli::Env < Kamal::Cli::Base
       end
 
       on(KAMAL.traefik_hosts) do
-        traefik_static_config = KAMAL.traefik_static.static_config
+        execute *KAMAL.traefik.make_env_directory
-        execute *KAMAL.traefik_static.make_env_directory
+        upload! StringIO.new(KAMAL.traefik.env_file), KAMAL.traefik.host_env_file_path, mode: 400
-        upload! StringIO.new(traefik_static_config.env_file), traefik_static_config.host_env_file_path, mode: 400
       end
 
       on(KAMAL.accessory_hosts) do
@@ -39,7 +38,7 @@ class Kamal::Cli::Env < Kamal::Cli::Base
       end
 
       on(KAMAL.traefik_hosts) do
-        execute *KAMAL.traefik_static.remove_env_file
+        execute *KAMAL.traefik.remove_env_file
       end
 
       on(KAMAL.accessory_hosts) do

lib/kamal/cli/traefik.rb

@@ -4,8 +4,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
     mutating do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.registry.login
-        execute *KAMAL.traefik_static.ensure_config_directory
+        execute *KAMAL.traefik.start_or_run
-        execute *KAMAL.traefik_static.start_or_run
       end
     end
   end
@@ -17,10 +16,9 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
       on(KAMAL.traefik_hosts, in: options[:rolling] ? :sequence : :parallel) do
         execute *KAMAL.auditor.record("Rebooted traefik"), verbosity: :debug
         execute *KAMAL.registry.login
-        execute *KAMAL.traefik_static.stop
+        execute *KAMAL.traefik.stop
-        execute *KAMAL.traefik_static.remove_container
+        execute *KAMAL.traefik.remove_container
-        execute *KAMAL.traefik_static.ensure_config_directory
+        execute *KAMAL.traefik.run
-        execute *KAMAL.traefik_static.run
       end
     end
   end
@@ -30,7 +28,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
     mutating do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.auditor.record("Started traefik"), verbosity: :debug
-        execute *KAMAL.traefik_static.start
+        execute *KAMAL.traefik.start
       end
     end
   end
@@ -40,7 +38,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
     mutating do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.auditor.record("Stopped traefik"), verbosity: :debug
-        execute *KAMAL.traefik_static.stop
+        execute *KAMAL.traefik.stop
       end
     end
   end
@@ -55,7 +53,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
 
   desc "details", "Show details about Traefik container from servers"
   def details
-    on(KAMAL.traefik_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.traefik_static.info), type: "Traefik" }
+    on(KAMAL.traefik_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.traefik.info), type: "Traefik" }
   end
 
   desc "logs", "Show log lines from Traefik on servers"
@@ -69,15 +67,15 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
     if options[:follow]
       run_locally do
         info "Following logs on #{KAMAL.primary_host}..."
-        info KAMAL.traefik_static.follow_logs(host: KAMAL.primary_host, grep: grep)
+        info KAMAL.traefik.follow_logs(host: KAMAL.primary_host, grep: grep)
-        exec KAMAL.traefik_static.follow_logs(host: KAMAL.primary_host, grep: grep)
+        exec KAMAL.traefik.follow_logs(host: KAMAL.primary_host, grep: grep)
       end
     else
       since = options[:since]
       lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
       on(KAMAL.traefik_hosts) do |host|
-        puts_by_host host, capture(*KAMAL.traefik_static.logs(since: since, lines: lines, grep: grep)), type: "Traefik"
+        puts_by_host host, capture(*KAMAL.traefik.logs(since: since, lines: lines, grep: grep)), type: "Traefik"
       end
     end
   end
@@ -96,7 +94,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
     mutating do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.auditor.record("Removed traefik container"), verbosity: :debug
-        execute *KAMAL.traefik_static.remove_container
+        execute *KAMAL.traefik.remove_container
       end
     end
   end
@@ -106,7 +104,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
     mutating do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.auditor.record("Removed traefik image"), verbosity: :debug
-        execute *KAMAL.traefik_static.remove_image
+        execute *KAMAL.traefik.remove_image
       end
     end
   end

lib/kamal/commander.rb

@@ -81,7 +81,7 @@ class Kamal::Commander
 
 
   def app(role: nil)
-    Kamal::Commands::App.new(config, role: role || config.roles.first.name)
+    Kamal::Commands::App.new(config, role: role)
   end
 
   def accessory(name)
@@ -124,12 +124,8 @@ class Kamal::Commander
     @server ||= Kamal::Commands::Server.new(config)
   end
 
-  def traefik_static
+  def traefik
-    @traefik_static ||= Kamal::Commands::Traefik::Static.new(config)
+    @traefik ||= Kamal::Commands::Traefik.new(config)
-  end
-
-  def traefik_dynamic(role: nil)
-    Kamal::Commands::Traefik::Dynamic.new(config, role: role || config.roles.first.name)
   end
 
   def with_verbosity(level)

lib/kamal/commands/accessory.rb

@@ -86,10 +86,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     end
   end
 
-  def make_directory_for(remote_file)
-    make_directory Pathname.new(remote_file).dirname.to_s
-  end
-
   def remove_service_directory
     [ :rm, "-rf", service_name ]
   end

lib/kamal/commands/app.rb

@@ -1,15 +1,12 @@
 class Kamal::Commands::App < Kamal::Commands::Base
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 
-  attr_reader :role
+  attr_reader :role, :role_config
 
   def initialize(config, role: nil)
     super(config)
     @role = role
-  end
+    @role_config = config.role(self.role)
-
-  def start_or_run(hostname: nil)
-    combine start, run(hostname: hostname), by: "||"
   end
 
   def run(hostname: nil)
@@ -23,6 +20,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
       *role_config.health_check_args,
       *config.logging_args,
       *config.volume_args,
+      *role_config.asset_volume_args,
       *role_config.label_args,
       *role_config.option_args,
       config.absolute_image,
@@ -47,10 +45,6 @@ class Kamal::Commands::App < Kamal::Commands::Base
     docker :ps, *filter_args
   end
 
-  def ip_address(version:)
-    docker :inspect, "-f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}'", container_name(version)
-  end
-
 
   def logs(since: nil, lines: nil, grep: nil)
     pipe \
@@ -112,7 +106,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
   def list_versions(*docker_args, statuses: nil)
     pipe \
       docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
-      %(while read line; do echo ${line##{role_config.full_name}-}; done) # Extract SHA from "service-role-dest-SHA"
+      %(while read line; do echo ${line##{role_config.container_prefix}-}; done) # Extract SHA from "service-role-dest-SHA"
   end
 
   def list_containers
@@ -150,26 +144,74 @@ class Kamal::Commands::App < Kamal::Commands::Base
   end
 
   def make_env_directory
-    make_directory config.role(role).host_env_directory
+    make_directory role_config.host_env_directory
   end
 
   def remove_env_file
-    [:rm, "-f", config.role(role).host_env_file_path]
+    [:rm, "-f", role_config.host_env_file_path]
   end
 
-  def service_role_dest
+  def cord(version:)
-    [config.service, role, config.destination].compact.join("-")
+    pipe \
+      docker(:inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", container_name(version)),
+      [:awk, "'$2 == \"#{role_config.cord_volume.container_path}\" {print $1}'"]
+  end
+
+  def tie_cord(cord)
+    create_empty_file(cord)
+  end
+
+  def cut_cord(cord)
+    remove_directory(cord)
+  end
+
+  def extract_assets
+    asset_container = "#{role_config.container_prefix}-assets"
+
+    combine \
+      make_directory(role_config.asset_extracted_path),
+      [*docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true"],
+      docker(:run, "--name", asset_container, "--detach", "--rm", config.latest_image, "sleep infinity"),
+      docker(:cp, "-L", "#{asset_container}:#{role_config.asset_path}/.", role_config.asset_extracted_path),
+      docker(:stop, "-t 1", asset_container),
+      by: "&&"
+  end
+
+  def sync_asset_volumes(old_version: nil)
+    new_extracted_path, new_volume_path = role_config.asset_extracted_path(config.version), role_config.asset_volume.host_path
+    if old_version.present?
+      old_extracted_path, old_volume_path = role_config.asset_extracted_path(old_version), role_config.asset_volume(old_version).host_path
+    end
+
+    commands = [make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path)]
+
+    if old_version.present?
+      commands << copy_contents(new_extracted_path, old_volume_path)
+      commands << copy_contents(old_extracted_path, new_volume_path)
+    end
+
+    chain *commands
+  end
+
+  def cleanup_assets
+    chain \
+      find_and_remove_older_siblings(role_config.asset_extracted_path),
+      find_and_remove_older_siblings(role_config.asset_volume_path)
   end
 
   private
     def container_name(version = nil)
-      [ role_config.full_name, version || config.version ].compact.join("-")
+      [ role_config.container_prefix, version || config.version ].compact.join("-")
     end
 
     def filter_args(statuses: nil)
       argumentize "--filter", filters(statuses: statuses)
     end
 
+    def service_role_dest
+      [config.service, role, config.destination].compact.join("-")
+    end
+
     def filters(statuses: nil)
       [ "label=service=#{config.service}" ].tap do |filters|
         filters << "label=destination=#{config.destination}" if config.destination
@@ -180,7 +222,18 @@ class Kamal::Commands::App < Kamal::Commands::Base
       end
     end
 
-    def role_config
+    def find_and_remove_older_siblings(path)
-      @role_config ||= config.role(self.role)
+      [
+        :find,
+        Pathname.new(path).dirname,
+        "-maxdepth 1",
+        "-name", "'#{role_config.container_prefix}-*'",
+        "!", "-name", Pathname.new(path).basename,
+        "-exec rm -rf \"{}\" +"
+      ]
+    end
+
+    def copy_contents(source, destination)
+      [ :cp, "-rn", "#{source}/*", destination ]
     end
 end

lib/kamal/commands/base.rb

@@ -34,6 +34,10 @@ module Kamal::Commands
       [ :mkdir, "-p", path ]
     end
 
+    def remove_directory(path)
+      [ :rm, "-r", path ]
+    end
+
     private
       def combine(*commands, by: "&&")
         commands
@@ -69,5 +73,11 @@ module Kamal::Commands
       def tags(**details)
         Kamal::Tags.from_config(config, **details)
       end
+
+      def create_empty_file(file)
+        chain \
+          make_directory_for(file),
+          [:touch, file]
+      end
   end
 end

lib/kamal/commands/healthcheck.rb

@@ -1,5 +1,4 @@
 class Kamal::Commands::Healthcheck < Kamal::Commands::Base
-  EXPOSED_PORT = 3999
 
   def run
     web = config.role(:web)
@@ -7,11 +6,11 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
     docker :run,
       "--detach",
       "--name", container_name_with_version,
-      "--publish", "#{EXPOSED_PORT}:#{config.healthcheck["port"]}",
+      "--publish", "#{exposed_port}:#{config.healthcheck["port"]}",
       "--label", "service=#{container_name}",
       "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
       *web.env_args,
-      *web.health_check_args,
+      *web.health_check_args(cord: false),
       *config.volume_args,
       *web.option_args,
       config.absolute_image,
@@ -52,6 +51,10 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
     end
 
     def health_url
-      "http://localhost:#{EXPOSED_PORT}#{config.healthcheck["path"]}"
+      "http://localhost:#{exposed_port}#{config.healthcheck["path"]}"
+    end
+
+    def exposed_port
+      config.healthcheck["exposed_port"]
     end
 end

lib/kamal/commands/server.rb

@@ -1,5 +1,5 @@
 class Kamal::Commands::Server < Kamal::Commands::Base
   def ensure_run_directory
-    make_directory config.run_directory
+    [:mkdir, "-p", config.run_directory]
   end
 end

lib/kamal/commands/traefik.rb

@@ -0,0 +1,122 @@
+class Kamal::Commands::Traefik < Kamal::Commands::Base
+  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
+
+  DEFAULT_IMAGE = "traefik:v2.9"
+  CONTAINER_PORT = 80
+  DEFAULT_ARGS = {
+    'log.level' => 'DEBUG'
+  }
+
+  def run
+    docker :run, "--name traefik",
+      "--detach",
+      "--restart", "unless-stopped",
+      *publish_args,
+      "--volume", "/var/run/docker.sock:/var/run/docker.sock",
+      *env_args,
+      *config.logging_args,
+      *label_args,
+      *docker_options_args,
+      image,
+      "--providers.docker",
+      *cmd_option_args
+  end
+
+  def start
+    docker :container, :start, "traefik"
+  end
+
+  def stop
+    docker :container, :stop, "traefik"
+  end
+
+  def start_or_run
+    combine start, run, by: "||"
+  end
+
+  def info
+    docker :ps, "--filter", "name=^traefik$"
+  end
+
+  def logs(since: nil, lines: nil, grep: nil)
+    pipe \
+      docker(:logs, "traefik", (" --since #{since}" if since), (" --tail #{lines}" if lines), "--timestamps", "2>&1"),
+      ("grep '#{grep}'" if grep)
+  end
+
+  def follow_logs(host:, grep: nil)
+    run_over_ssh pipe(
+      docker(:logs, "traefik", "--timestamps", "--tail", "10", "--follow", "2>&1"),
+      (%(grep "#{grep}") if grep)
+    ).join(" "), host: host
+  end
+
+  def remove_container
+    docker :container, :prune, "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
+  end
+
+  def remove_image
+    docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
+  end
+
+  def port
+    "#{host_port}:#{CONTAINER_PORT}"
+  end
+
+  def env_file
+    env_file_with_secrets config.traefik.fetch("env", {})
+  end
+
+  def host_env_file_path
+    File.join host_env_directory, "traefik.env"
+  end
+
+  def make_env_directory
+    make_directory(host_env_directory)
+  end
+
+  def remove_env_file
+    [:rm, "-f", host_env_file_path]
+  end
+
+  private
+    def publish_args
+      argumentize "--publish", port unless config.traefik["publish"] == false
+    end
+
+    def label_args
+      argumentize "--label", labels
+    end
+
+    def env_args
+      argumentize "--env-file", host_env_file_path
+    end
+
+    def host_env_directory
+      File.join config.host_env_directory, "traefik"
+    end
+
+    def labels
+      config.traefik["labels"] || []
+    end
+
+    def image
+      config.traefik.fetch("image") { DEFAULT_IMAGE }
+    end
+
+    def docker_options_args
+      optionize(config.traefik["options"] || {})
+    end
+
+    def cmd_option_args
+      if args = config.traefik["args"]
+        optionize DEFAULT_ARGS.merge(args), with: "="
+      else
+        optionize DEFAULT_ARGS, with: "="
+      end
+    end
+
+    def host_port
+      config.traefik["host_port"] || CONTAINER_PORT
+    end
+end

lib/kamal/commands/traefik/dynamic.rb

@@ -1,43 +0,0 @@
-class Kamal::Commands::Traefik::Dynamic < Kamal::Commands::Base
-  attr_reader :static_config, :dynamic_config
-
-  def initialize(config, role: nil)
-    super(config)
-    @static_config = Kamal::Configuration::Traefik::Static.new(config: config)
-    @dynamic_config = Kamal::Configuration::Traefik::Dynamic.new(config: config, role: role)
-  end
-
-  def run_id
-    pipe \
-      [:docker, :exec, :traefik, :wget, "-qSO", "/dev/null", "http://localhost:#{Kamal::Configuration::Traefik::Static::CONTAINER_PORT}#{config.healthcheck["path"]}", "2>&1"],
-      [:grep, "-i", Kamal::Configuration::Traefik::Dynamic::RUN_ID_HEADER],
-      [:cut, "-d ' ' -f 4"]
-  end
-
-  def write_config(ip_address:)
-    # Write to tmp then mv for an atomic copy. If you write directly traefik sees an empty file
-    # and removes the service before picking up the new config.
-    temp_config_file = "/tmp/kamal-traefik-config-#{rand(10000000)}"
-    chain \
-      write([:echo, dynamic_config.config(ip_address: ip_address).to_yaml.shellescape], temp_config_file),
-      [:mv, temp_config_file, host_file]
-  end
-
-  def remove_config
-    [:rm, host_file]
-  end
-
-  def boot_check?
-    dynamic_config.boot_check?
-  end
-
-  def config_run_id
-    dynamic_config.run_id
-  end
-
-  private
-    def host_file
-      "#{static_config.host_directory}/#{dynamic_config.host_file}"
-    end
-end
-

lib/kamal/commands/traefik/static.rb

@@ -1,70 +0,0 @@
-class Kamal::Commands::Traefik::Static < Kamal::Commands::Base
-  attr_reader :static_config, :dynamic_config
-
-  def initialize(config, role: nil)
-    super(config)
-    @static_config = Kamal::Configuration::Traefik::Static.new(config: config)
-  end
-
-  def run
-    docker :run, static_config.docker_args, static_config.image, static_config.traefik_args
-  end
-
-  def start
-    docker :container, :start, "traefik"
-  end
-
-  def stop
-    docker :container, :stop, "traefik"
-  end
-
-  def start_or_run
-    combine start, run, by: "||"
-  end
-
-  def info
-    docker :ps, "--filter", "name=^traefik$"
-  end
-
-  def logs(since: nil, lines: nil, grep: nil)
-    pipe \
-      docker(:logs, "traefik", (" --since #{since}" if since), (" --tail #{lines}" if lines), "--timestamps", "2>&1"),
-      ("grep '#{grep}'" if grep)
-  end
-
-  def follow_logs(host:, grep: nil)
-    run_over_ssh pipe(
-      docker(:logs, "traefik", "--timestamps", "--tail", "10", "--follow", "2>&1"),
-      (%(grep "#{grep}") if grep)
-    ).join(" "), host: host
-  end
-
-  def remove_container
-    docker :container, :prune, "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
-  end
-
-  def remove_image
-    docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
-  end
-
-  def port
-    "#{host_port}:#{CONTAINER_PORT}"
-  end
-
-  def make_env_directory
-    make_directory(static_config.host_env_directory)
-  end
-
-  def remove_env_file
-    [:rm, "-f", static_config.host_env_file_path]
-  end
-
-  def ensure_config_directory
-    make_directory(static_config.host_directory)
-  end
-
-  def docker_entrypoint_args
-    docker :inspect, "-f '{{index .Args 1 }}'", :traefik
-  end
-end
-

lib/kamal/configuration.rb

@@ -7,7 +7,7 @@ require "net/ssh/proxy/jump"
 
 class Kamal::Configuration
   delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_accessor :destination
   attr_accessor :raw_config
@@ -61,6 +61,14 @@ class Kamal::Configuration
     raw_config.run_directory || ".kamal"
   end
 
+  def run_directory_as_docker_volume
+    if Pathname.new(run_directory).absolute?
+      run_directory
+    else
+      File.join "$(pwd)", run_directory
+    end
+  end
+
 
   def roles
     @roles ||= role_names.collect { |role_name| Role.new(role_name, config: self) }
@@ -141,7 +149,7 @@ class Kamal::Configuration
 
 
   def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7 }.merge(raw_config.healthcheck || {})
+    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }.merge(raw_config.healthcheck || {})
   end
 
   def readiness_delay
@@ -153,7 +161,7 @@ class Kamal::Configuration
   end
 
   def valid?
-    ensure_required_keys_present && ensure_valid_kamal_version && ensure_no_traefik_labels
+    ensure_required_keys_present && ensure_valid_kamal_version
   end
 
 
@@ -199,6 +207,14 @@ class Kamal::Configuration
     "#{run_directory}/env"
   end
 
+  def run_id
+    @run_id ||= SecureRandom.hex(16)
+  end
+
+  def asset_path
+    raw_config.asset_path
+  end
+
   private
     # Will raise ArgumentError if any required config keys are missing
     def ensure_required_keys_present
@@ -231,17 +247,6 @@ class Kamal::Configuration
       true
     end
 
-    def ensure_no_traefik_labels
-      # The switch to a traefik file provider means that traefik labels on app containers are ignored
-      # We'll raise an error and suggest moving them
-
-      if roles.any? { |role| role.labels.keys.any? { |label| label.start_with?("traefik.") } }
-        raise ArgumentError, "Traefik is not configured to read labels, move traefik config to dynamic:"
-      end
-
-      true
-    end
-
 
     def role_names
       raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort

lib/kamal/configuration/role.rb

@@ -1,10 +1,11 @@
 class Kamal::Configuration::Role
+  CORD_FILE = "cord"
   delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
 
   attr_accessor :name
 
   def initialize(name, config:)
-    @name, @config = name.inquiry, config
+   @name, @config = name.inquiry, config
   end
 
   def primary_host
@@ -16,7 +17,7 @@ class Kamal::Configuration::Role
   end
 
   def labels
-    default_labels.merge(custom_labels)
+    default_labels.merge(traefik_labels).merge(custom_labels)
   end
 
   def label_args
@@ -47,28 +48,64 @@ class Kamal::Configuration::Role
     argumentize "--env-file", host_env_file_path
   end
 
-  def health_check_args
+  def asset_volume_args
+    asset_volume&.docker_args
+  end
+
+  def health_check_args(cord: true)
     if health_check_cmd.present?
-      optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval })
+      if cord && uses_cord?
+        optionize({ "health-cmd" => health_check_cmd_with_cord, "health-interval" => health_check_interval })
+          .concat(cord_volume.docker_args)
+      else
+        optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval })
+      end
     else
       []
     end
   end
 
   def health_check_cmd
-    options = specializations["healthcheck"] || {}
+    health_check_options["cmd"] || http_health_check(port: health_check_options["port"], path: health_check_options["path"])
-    options = config.healthcheck.merge(options) if running_traefik?
+  end
 
-    options["cmd"] || http_health_check(port: options["port"], path: options["path"])
+  def health_check_cmd_with_cord
+    "(#{health_check_cmd}) && (stat #{cord_container_file} > /dev/null || exit 1)"
   end
 
   def health_check_interval
-    options = specializations["healthcheck"] || {}
+    health_check_options["interval"] || "1s"
-    options = config.healthcheck.merge(options) if running_traefik?
-
-    options["interval"] || "1s"
   end
 
+  def uses_cord?
+    running_traefik? && cord_volume && health_check_cmd.present?
+  end
+
+  def cord_host_directory
+    File.join config.run_directory_as_docker_volume, "cords", [container_prefix, config.run_id].join("-")
+  end
+
+  def cord_volume
+    if (cord = health_check_options["cord"])
+      @cord_volume ||= Kamal::Configuration::Volume.new \
+        host_path: File.join(config.run_directory, "cords", [container_prefix, config.run_id].join("-")),
+        container_path: cord
+    end
+  end
+
+  def cord_host_file
+    File.join cord_volume.host_path, CORD_FILE
+  end
+
+  def cord_container_directory
+    health_check_options.fetch("cord", nil)
+  end
+
+  def cord_container_file
+    File.join cord_volume.container_path, CORD_FILE
+  end
+
+
   def cmd
     specializations["cmd"]
   end
@@ -82,22 +119,39 @@ class Kamal::Configuration::Role
   end
 
   def running_traefik?
-    name.web? || (specializations["traefik"] != nil && specializations["traefik"] != false)
+    name.web? || specializations["traefik"]
   end
 
-  def traefik
+  def container_name(version = nil)
-    case specializations["traefik"]
+    [ container_prefix, version || config.version ].compact.join("-")
-    when NilClass, TrueClass, FalseClass
-      {}
-    else
-      specializations["traefik"]
-    end
   end
 
-  def full_name
+  def container_prefix
     [ config.service, name, config.destination ].compact.join("-")
   end
 
+  def asset_path
+    specializations["asset_path"] || config.asset_path
+  end
+
+  def assets?
+    asset_path.present? && running_traefik?
+  end
+
+  def asset_volume(version = nil)
+    if assets?
+      Kamal::Configuration::Volume.new \
+        host_path: asset_volume_path(version), container_path: asset_path
+    end
+  end
+
+  def asset_extracted_path(version = nil)
+    File.join config.run_directory, "assets", "extracted", container_name(version)
+  end
+
+  def asset_volume_path(version = nil)
+    File.join config.run_directory, "assets", "volumes", container_name(version)
+  end
 
   private
     attr_accessor :config
@@ -119,6 +173,26 @@ class Kamal::Configuration::Role
       end
     end
 
+    def traefik_labels
+      if running_traefik?
+        {
+          # Setting a service property ensures that the generated service name will be consistent between versions
+          "traefik.http.services.#{traefik_service}.loadbalancer.server.scheme" => "http",
+
+          "traefik.http.routers.#{traefik_service}.rule" => "PathPrefix(`/`)",
+          "traefik.http.middlewares.#{traefik_service}-retry.retry.attempts" => "5",
+          "traefik.http.middlewares.#{traefik_service}-retry.retry.initialinterval" => "500ms",
+          "traefik.http.routers.#{traefik_service}.middlewares" => "#{traefik_service}-retry@docker"
+        }
+      else
+        {}
+      end
+    end
+
+    def traefik_service
+      [ config.service, name, config.destination ].compact.join("-")
+    end
+
     def custom_labels
       Hash.new.tap do |labels|
         labels.merge!(config.labels) if config.labels.present?
@@ -158,4 +232,12 @@ class Kamal::Configuration::Role
     def http_health_check(port:, path:)
       "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present?
     end
+
+    def health_check_options
+      @health_check_options ||= begin
+        options = specializations["healthcheck"] || {}
+        options = config.healthcheck.merge(options) if running_traefik?
+        options
+      end
+    end
 end

lib/kamal/configuration/traefik/dynamic.rb

@@ -1,66 +0,0 @@
-class Kamal::Configuration::Traefik::Dynamic
-  RUN_ID_HEADER = "X-Kamal-Run-ID"
-
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
-
-  attr_reader :traefik_config, :role_config, :role_traefik_config
-
-  def initialize(config:, role:)
-    @traefik_config = config.traefik || {}
-    @role_config = config.role(role)
-    @role_traefik_config = role_config&.traefik || {}
-  end
-
-  def host_file
-    "#{role_config.full_name}.yml"
-  end
-
-  def config(ip_address:)
-    default_config(ip_address:).deep_merge!(custom_config)
-  end
-
-  def boot_check?
-    role_traefik_config.fetch("boot_check") { traefik_config.fetch("boot_check", true) }
-  end
-
-  def run_id
-    @run_id ||= SecureRandom.hex(16)
-  end
-
-  private
-    def default_config(ip_address:)
-      run_id_header_middleware = "#{role_config.full_name}-id-header"
-
-      {
-        "http" => {
-          "routers" => {
-            role_config.full_name => {
-              "rule" => "PathPrefix(`/`)",
-              "middlewares" => [ run_id_header_middleware ],
-              "service" => role_config.full_name
-            }
-          },
-          "services" => {
-            role_config.full_name => {
-              "loadbalancer" => {
-                "servers" => [ { "url" => "http://#{ip_address}:80" } ]
-              }
-            }
-          },
-          "middlewares" => {
-            run_id_header_middleware => {
-              "headers" => {
-                "customresponseheaders" => {
-                  RUN_ID_HEADER => run_id
-                }
-              }
-            }
-          }
-        }
-      }
-    end
-
-    def custom_config
-      traefik_config.fetch("dynamic", {}).deep_merge(role_traefik_config.fetch("dynamic", {}))
-    end
-end

lib/kamal/configuration/traefik/static.rb

@@ -1,84 +0,0 @@
-class Kamal::Configuration::Traefik::Static
-  CONTAINER_PORT = 80
-  DEFAULT_IMAGE = "traefik:v2.9"
-  CONFIG_DIRECTORY = "/var/run/traefik-config"
-  DEFAULT_ARGS = {
-    "providers.docker": true, # Obsolete now but required for zero-downtime upgrade from previous versions
-    "providers.file.directory" =>  "/var/run/traefik-config",
-    "providers.file.watch": true,
-    "log.level" => "DEBUG",
-  }
-
-  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
-
-  attr_reader :config, :traefik_config
-
-  def initialize(config:)
-    @config = config
-    @traefik_config = config.traefik || {}
-  end
-
-  def docker_args
-    [
-      "--name traefik",
-      "--detach",
-      "--restart", "unless-stopped",
-      *publish_args,
-      "--volume", "/var/run/docker.sock:/var/run/docker.sock",
-      "--volume", "#{host_directory}:#{CONFIG_DIRECTORY}",
-      *env_args,
-      *config.logging_args,
-      *label_args,
-      *docker_options_args
-    ]
-  end
-
-  def image
-    traefik_config.fetch("image") { DEFAULT_IMAGE }
-  end
-
-  def traefik_args
-    optionize DEFAULT_ARGS.merge(traefik_config.fetch("args", {})), with: "="
-  end
-
-  def host_directory
-    if Pathname.new(config.run_directory).absolute?
-      "#{config.run_directory}/traefik-config"
-    else
-      "$(pwd)/#{config.run_directory}/traefik-config"
-    end
-  end
-
-  def host_env_file_path
-    File.join host_env_directory, "traefik.env"
-  end
-
-  def host_env_directory
-    File.join config.host_env_directory, "traefik"
-  end
-
-  def env_file
-    env_file_with_secrets config.traefik.fetch("env", {})
-  end
-
-  private
-    def host_port
-      traefik_config.fetch("host_port", CONTAINER_PORT)
-    end
-
-    def publish_args
-      argumentize "--publish", "#{host_port}:#{CONTAINER_PORT}" unless traefik_config["publish"] == false
-    end
-
-    def env_args
-      argumentize "--env-file", host_env_file_path
-    end
-
-    def label_args
-      argumentize "--label", traefik_config.fetch("labels", [])
-    end
-
-    def docker_options_args
-      optionize(traefik_config["options"] || {})
-    end
-end

lib/kamal/configuration/volume.rb

@@ -0,0 +1,22 @@
+class Kamal::Configuration::Volume
+  attr_reader :host_path, :container_path
+  delegate :argumentize, to: Kamal::Utils
+
+  def initialize(host_path:, container_path:)
+    @host_path = host_path
+    @container_path = container_path
+  end
+
+  def docker_args
+    argumentize "--volume", "#{host_path_for_docker_volume}:#{container_path}"
+  end
+
+  private
+    def host_path_for_docker_volume
+      if Pathname.new(host_path).absolute?
+        host_path
+      else
+        File.join "$(pwd)", host_path
+      end
+    end
+end

lib/kamal/utils.rb

@@ -115,25 +115,4 @@ module Kamal::Utils
 
     "#{key.to_s}=#{value.to_s}\n"
   end
-
-  def poll(max_attempts:, exception:, &block)
-    attempt = 1
-
-    begin
-      block.call
-    rescue exception => e
-      if attempt <= max_attempts
-        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-        sleep attempt
-        attempt += 1
-        retry
-      else
-        raise
-      end
-    end
-  end
-
-  def info(message)
-    SSHKit.config.output.info(message)
-  end
 end

lib/kamal/utils/healthcheck_poller.rb

@@ -1,22 +1,64 @@
 class Kamal::Utils::HealthcheckPoller
-  TRAEFIK_HEALTHY_DELAY = 2
+  TRAEFIK_UPDATE_DELAY = 2
 
   class HealthcheckError < StandardError; end
 
   class << self
     def wait_for_healthy(pause_after_ready: false, &block)
-      Kamal::Utils.poll(max_attempts: KAMAL.config.healthcheck["max_attempts"], exception: HealthcheckError) do
+      attempt = 1
+      max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+      begin
         case status = block.call
         when "healthy"
-          sleep TRAEFIK_HEALTHY_DELAY if pause_after_ready
+          sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
         when "running" # No health check configured
           sleep KAMAL.config.readiness_delay if pause_after_ready
         else
           raise HealthcheckError, "container not ready (#{status})"
         end
-
+      rescue HealthcheckError => e
-        SSHKit.config.output.info "Container is healthy!"
+        if attempt <= max_attempts
+          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+          sleep attempt
+          attempt += 1
+          retry
+        else
+          raise
+        end
       end
+
+      info "Container is healthy!"
     end
+
+    def wait_for_unhealthy(pause_after_ready: false, &block)
+      attempt = 1
+      max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+      begin
+        case status = block.call
+        when "unhealthy"
+          sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
+        else
+          raise HealthcheckError, "container not unhealthy (#{status})"
+        end
+      rescue HealthcheckError => e
+        if attempt <= max_attempts
+          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+          sleep attempt
+          attempt += 1
+          retry
+        else
+          raise
+        end
+      end
+
+      info "Container is unhealthy!"
+    end
+
+    private
+      def info(message)
+        SSHKit.config.output.info(message)
+      end
   end
 end

lib/kamal/utils/switch_poller.rb

@@ -1,17 +0,0 @@
-class Kamal::Utils::SwitchPoller
-  class SwitchError < StandardError; end
-
-  class << self
-    TRAEFIK_SWITCH_DELAY = 2
-    def wait_for_switch(traefik_dynamic, &block)
-      if traefik_dynamic.boot_check?
-        Kamal::Utils.poll(max_attempts: 5, exception: SwitchError) do
-          polled_run_id = block.call
-          raise SwitchError, "Waiting for #{traefik_dynamic.config_run_id}, currently #{polled_run_id}" unless polled_run_id == traefik_dynamic.config_run_id
-        end
-      else
-        sleep TRAEFIK_SWITCH_DELAY
-      end
-    end
-  end
-end

test/cli/app_test.rb

@@ -5,21 +5,43 @@ class CliAppTest < CliTestCase
     stub_running
     run_command("boot").tap do |output|
       assert_match "docker tag dhh/app:latest dhh/app:latest", output
-      assert_match /docker rename app-web-latest app-web-latest_replaced_[0-9a-f]{16}/, output
       assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} /, output
       assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
     end
   end
 
   test "boot will rename if same version is already running" do
+    Object.any_instance.stubs(:sleep)
     run_command("details") # Preheat Kamal const
 
-    stub_running
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678") # running version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .returns("cordfile") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("unhealthy") # old version unhealthy
+
     run_command("boot").tap do |output|
       assert_match /Renaming container .* to .* as already deployed on 1.1.1.1/, output # Rename
+      assert_match /docker rename app-web-latest app-web-latest_replaced_[0-9a-f]{16}/, output
       assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} /, output
       assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
     end
+  ensure
+    Thread.report_on_exception = true
   end
 
   test "boot uses group strategy when specified" do
@@ -32,27 +54,9 @@ class CliAppTest < CliTestCase
     run_command("boot", config: :with_boot_strategy)
   end
 
-  test "boot without traefik file provider raises exception" do
-    Thread.report_on_exception = false
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{index .Args 1 }}'", :traefik)
-      .returns("[--providers.docker --log.level=DEBUG --accesslog --accesslog.format=json]").at_least_once
-
-    assert_raises(SSHKit::Runner::ExecuteError, "Exception while executing on host 1.1.1.1: File provider not enabled, you'll need to run `kamal traefik reboot` to deploy") do
-      run_command("boot")
-    end
-  ensure
-    Thread.report_on_exception = true
-  end
-
   test "boot errors leave lock in place" do
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
 
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{index .Args 1 }}'", :traefik)
-      .returns("[--providers.docker --providers.file.directory=/var/run/traefik-config --providers.file.watch --log.level=DEBUG --accesslog --accesslog.format=json]").at_least_once
-
     Kamal::Cli::App.any_instance.expects(:using_version).raises(RuntimeError)
 
     assert !KAMAL.holding_lock?
@@ -63,13 +67,6 @@ class CliAppTest < CliTestCase
   end
 
   test "start" do
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with { |*args| args == [ :docker, :inspect, "-f '{{index .Args 1 }}'", :traefik ] }
-      .returns("[--providers.docker --providers.file.directory=/var/run/traefik-config --providers.file.watch --log.level=DEBUG --accesslog --accesslog.format=json]").at_least_once
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with { |*args| args != [ :docker, :inspect, "-f '{{index .Args 1 }}'", :traefik ] }
-      .returns("").at_least_once
-
     run_command("start").tap do |output|
       assert_match "docker start app-web-999", output
     end
@@ -136,7 +133,7 @@ class CliAppTest < CliTestCase
 
   test "exec" do
     run_command("exec", "ruby -v").tap do |output|
-      assert_match "docker run --rm --env-file .kamal/env/roles/app-web.env dhh/app:latest ruby -v", output
+      assert_match "docker run --rm dhh/app:latest ruby -v", output
     end
   end
 
@@ -192,26 +189,16 @@ class CliAppTest < CliTestCase
     end
 
     def stub_running
-      SecureRandom.stubs(:hex).with(16).returns("12345678901234567890123456789012")
+      Object.any_instance.stubs(:sleep)
-      SecureRandom.stubs(:hex).with(6).returns("123456789012")
-      SecureRandom.stubs(:hex).with(8).returns("1234567890123456")
 
       SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
 
-      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :inspect, "-f '{{index .Args 1 }}'", :traefik)
-        .returns("[--providers.docker --providers.file.directory=/var/run/traefik-config --providers.file.watch --log.level=DEBUG --accesslog --accesslog.format=json]").at_least_once
-
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
         .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
         .returns("running") # health check
 
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :inspect, "-f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}'", "app-web-latest")
+        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-        .returns("172.17.0.3").at_least_once
+        .returns("unhealthy") # health check
-
-      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :exec, :traefik, :wget, "-qSO", "/dev/null", "http://localhost:80/up", "2>&1", "|", :grep, "-i", "X-Kamal-Run-ID", "|", :cut, "-d ' ' -f 4")
-        .returns("12345678901234567890123456789012").at_least_once
     end
 end

test/cli/healthcheck_test.rb

@@ -5,7 +5,8 @@ class CliHealthcheckTest < CliTestCase
     # Prevent expected failures from outputting to terminal
     Thread.report_on_exception = false
 
-    Object.any_instance.stubs(:sleep) # No sleeping when retrying
+    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
@@ -28,15 +29,13 @@ class CliHealthcheckTest < CliTestCase
       assert_match "container not ready (unhealthy), retrying in 2s (attempt 2/7)...", output
       assert_match "Container is healthy!", output
     end
-  ensure
-    Thread.report_on_exception = true
   end
 
   test "perform failing to become healthy" do
     # Prevent expected failures from outputting to terminal
     Thread.report_on_exception = false
 
-    Object.any_instance.stubs(:sleep) # No sleeping when retrying
+    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
@@ -64,8 +63,6 @@ class CliHealthcheckTest < CliTestCase
       run_command("perform")
     end
     assert_match "container not ready (unhealthy)", exception.message
-  ensure
-    Thread.report_on_exception = true
   end
 
   private

test/cli/main_test.rb

@@ -66,7 +66,7 @@ class CliMainTest < CliTestCase
       .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*args| args[0..1] == [:mkdir, ".kamal/lock-app"] }
+      .with { |*arg| arg[0..1] == [:mkdir, ".kamal/lock-app"] }
       .raises(RuntimeError, "mkdir: cannot create directory ‘kamal_lock-app’: File exists")
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_debug)
@@ -75,8 +75,6 @@ class CliMainTest < CliTestCase
     assert_raises(Kamal::Cli::LockError) do
       run_command("deploy")
     end
-  ensure
-    Thread.report_on_exception = true
   end
 
   test "deploy error when locking" do
@@ -92,8 +90,6 @@ class CliMainTest < CliTestCase
     assert_raises(SSHKit::Runner::ExecuteError) do
       run_command("deploy")
     end
-  ensure
-    Thread.report_on_exception = true
   end
 
   test "deploy errors during outside section leave remove lock" do
@@ -177,17 +173,13 @@ class CliMainTest < CliTestCase
       assert_match /docker container ls --all --filter name=\^app-web-nonsense\$ --quiet/, output
       assert_match /The app version 'nonsense' is not available as a container/, output
     end
-  ensure
-    Thread.report_on_exception = true
   end
 
   test "rollback good version" do
-    SecureRandom.stubs(:hex).with(16).returns("12345678901234567890123456789012")
+    Object.any_instance.stubs(:sleep)
-    SecureRandom.stubs(:hex).with(6).returns("123456789012")
-
     [ "web", "workers" ].each do |role|
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :container, :ls, "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false)
+        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false)
         .returns("").at_least_once
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
         .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet")
@@ -201,25 +193,20 @@ class CliMainTest < CliTestCase
     end
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}'", "app-web-123")
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
-      .returns("172.17.0.3").at_least_once
+      .returns("corddirectory").at_least_once # health check
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{index .Args 1 }}'", :traefik)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-version-to-rollback$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-      .returns("[--providers.docker --providers.file.directory=/var/run/traefik-config --providers.file.watch --log.level=DEBUG --accesslog --accesslog.format=json]").at_least_once
+      .returns("unhealthy").at_least_once # health check
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :exec, :traefik, :wget, "-qSO", "/dev/null", "http://localhost:80/up", "2>&1", "|", :grep, "-i", "X-Kamal-Run-ID", "|", :cut, "-d ' ' -f 4")
-      .returns("12345678901234567890123456789012").at_least_once
 
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     hook_variables = { version: 123, service_version: "app@123", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "rollback" }
 
     run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
-      assert_match "Start container with version 123", output
       assert_hook_ran "pre-deploy", output, **hook_variables
       assert_match "docker tag dhh/app:123 dhh/app:latest", output
-      assert_match "docker start app-web-123", output
+      assert_match "docker run --detach --restart unless-stopped --name app-web-123", output
       assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running"
       assert_hook_ran "post-deploy", output, **hook_variables, runtime: "0"
     end
@@ -228,15 +215,10 @@ class CliMainTest < CliTestCase
   test "rollback without old version" do
     Kamal::Cli::Main.any_instance.stubs(:container_available?).returns(true)
 
-    Object.stubs(:sleep)
+    Kamal::Utils::HealthcheckPoller.stubs(:sleep)
-    SecureRandom.stubs(:hex).with(16).returns("12345678901234567890123456789012")
-    SecureRandom.stubs(:hex).with(6).returns("123456789012")
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{index .Args 1 }}'", :traefik)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
-      .returns("[--providers.docker --providers.file.directory=/var/run/traefik-config --providers.file.watch --log.level=DEBUG --accesslog --accesslog.format=json]").at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
       .returns("").at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
@@ -244,17 +226,9 @@ class CliMainTest < CliTestCase
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
       .returns("running").at_least_once # health check
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}'", "app-web-123")
-      .returns("172.17.0.3").at_least_once
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :exec, :traefik, :wget, "-qSO", "/dev/null", "http://localhost:80/up", "2>&1", "|", :grep, "-i", "X-Kamal-Run-ID", "|", :cut, "-d ' ' -f 4")
-      .returns("12345678901234567890123456789012").at_least_once
 
     run_command("rollback", "123").tap do |output|
-      assert_match "Start container with version 123", output
+      assert_match "docker run --detach --restart unless-stopped --name app-web-123", output
-      assert_match "docker start app-web-123 || docker run --detach --restart unless-stopped --name app-web-123", output
       assert_no_match "docker stop", output
     end
   end

test/cli/traefik_test.rb

@@ -4,7 +4,7 @@ class CliTraefikTest < CliTestCase
   test "boot" do
     run_command("boot").tap do |output|
       assert_match "docker login", output
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Configuration::Traefik::Static::DEFAULT_IMAGE} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\"", output
+      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
     end
   end
 
@@ -14,7 +14,7 @@ class CliTraefikTest < CliTestCase
     run_command("reboot").tap do |output|
       assert_match "docker container stop traefik", output
       assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik", output
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Configuration::Traefik::Static::DEFAULT_IMAGE} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\"", output
+      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
     end
   end
 

test/commands/app_test.rb

@@ -3,6 +3,7 @@ require "test_helper"
 class CommandsAppTest < ActiveSupport::TestCase
   setup do
     ENV["RAILS_MASTER_KEY"] = "456"
+    Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
 
     @config = { service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], env: { "secret" => [ "RAILS_MASTER_KEY" ] } }
   end
@@ -13,13 +14,13 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with hostname" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run(hostname: "myhost").join(" ")
   end
 
@@ -27,7 +28,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:volumes] = ["/local/path:/container/path" ]
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -35,7 +36,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "path" => "/healthz" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/healthz || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -43,7 +44,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "cmd" => "/bin/up" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/up\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -51,7 +52,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/healthy\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -66,7 +67,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -83,18 +84,6 @@ class CommandsAppTest < ActiveSupport::TestCase
       new_command.start.join(" ")
   end
 
-  test "start_or_run" do
-    assert_equal \
-      "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" dhh/app:999",
-      new_command.start_or_run.join(" ")
-  end
-
-  test "start_or_run with hostname" do
-    assert_equal \
-      "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" dhh/app:999",
-      new_command.start_or_run(hostname: "myhost").join(" ")
-  end
-
   test "stop" do
     assert_equal \
       "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop",
@@ -342,6 +331,20 @@ class CommandsAppTest < ActiveSupport::TestCase
     assert_equal "rm -f .kamal/env/roles/app-web.env", new_command.remove_env_file.join(" ")
   end
 
+  test "cord" do
+    assert_equal "docker inspect -f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}' app-web-123 | awk '$2 == \"/tmp/kamal-cord\" {print $1}'", new_command.cord(version: 123).join(" ")
+  end
+
+  test "tie cord" do
+    assert_equal "mkdir -p . ; touch cordfile", new_command.tie_cord("cordfile").join(" ")
+    assert_equal "mkdir -p corddir ; touch corddir/cordfile", new_command.tie_cord("corddir/cordfile").join(" ")
+    assert_equal "mkdir -p /corddir ; touch /corddir/cordfile", new_command.tie_cord("/corddir/cordfile").join(" ")
+  end
+
+  test "cut cord" do
+    assert_equal "rm -r corddir", new_command.cut_cord("corddir").join(" ")
+  end
+
   private
     def new_command(role: "web")
       Kamal::Commands::App.new(Kamal::Configuration.new(@config, destination: @destination, version: "999"), role: role)

test/commands/healthcheck_test.rb

@@ -40,8 +40,9 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
 
   test "run with custom options" do
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere" } } }
+    @config[:healthcheck] = { "exposed_port" => 4999 }
     assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 4999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
       new_command.run.join(" ")
   end
 

test/commands/traefik_test.rb

@@ -1,6 +1,6 @@
 require "test_helper"
 
-class CommandsTraefikStaticTest < ActiveSupport::TestCase
+class CommandsTraefikTest < ActiveSupport::TestCase
   setup do
     @image = "traefik:test"
 
@@ -18,72 +18,72 @@ class CommandsTraefikStaticTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["host_port"] = "8080"
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["publish"] = false
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with ports configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["options"] = {"publish" => %w[9000:9000 9001:9001]}
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with volumes configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json] }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with several options configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json], "publish" => %w[8080:8080], "memory" => "512m"}
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with labels configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["labels"] = { "traefik.http.routers.dashboard.service" => "api@internal", "traefik.http.routers.dashboard.middlewares" => "auth" }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.dashboard.service=\"api@internal\" --label traefik.http.routers.dashboard.middlewares=\"auth\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.dashboard.service=\"api@internal\" --label traefik.http.routers.dashboard.middlewares=\"auth\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with env configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -91,7 +91,7 @@ class CommandsTraefikStaticTest < ActiveSupport::TestCase
     @config.delete(:traefik)
 
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Configuration::Traefik::Static::DEFAULT_IMAGE} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"",
       new_command.run.join(" ")
   end
 
@@ -99,7 +99,7 @@ class CommandsTraefikStaticTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -107,7 +107,7 @@ class CommandsTraefikStaticTest < ActiveSupport::TestCase
     @config[:traefik]["args"]["log.level"] = "ERROR"
 
     assert_equal \
-    "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"ERROR\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+    "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"ERROR\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -177,6 +177,16 @@ class CommandsTraefikStaticTest < ActiveSupport::TestCase
       new_command.follow_logs(host: @config[:servers].first, grep: 'hello!')
   end
 
+  test "env_file" do
+    @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
+
+    assert_equal "EXAMPLE_API_KEY=456\n", new_command.env_file
+  end
+
+  test "host_env_file_path" do
+    assert_equal ".kamal/env/traefik/traefik.env", new_command.host_env_file_path
+  end
+
   test "make_env_directory" do
     assert_equal "mkdir -p .kamal/env/traefik", new_command.make_env_directory.join(" ")
   end
@@ -187,6 +197,6 @@ class CommandsTraefikStaticTest < ActiveSupport::TestCase
 
   private
     def new_command
-      Kamal::Commands::Traefik::Static.new(Kamal::Configuration.new(@config, version: "123"))
+      Kamal::Commands::Traefik.new(Kamal::Configuration.new(@config, version: "123"))
     end
 end

test/configuration/role_test.rb

@@ -42,7 +42,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
   end
 
   test "special label args for web" do
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\""], @config.role(:web).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args
   end
 
   test "custom labels" do
@@ -66,7 +66,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       c[:servers]["beta"] = { "traefik" => "true", "hosts" => [ "1.1.1.5" ] }
     })
 
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"" ], config.role(:beta).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "traefik.http.services.app-beta.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args
   end
 
   test "env overwritten by role" do
@@ -175,4 +175,23 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).host_env_file_path
   end
 
+  test "uses cord" do
+    assert @config_with_roles.role(:web).uses_cord?
+    assert !@config_with_roles.role(:workers).uses_cord?
+  end
+
+  test "cord host file" do
+    assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}/cord}, @config_with_roles.role(:web).cord_host_file
+  end
+
+  test "cord volume" do
+    assert_equal "/tmp/kamal-cord", @config_with_roles.role(:web).cord_volume.container_path
+    assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}}, @config_with_roles.role(:web).cord_volume.host_path
+    assert_equal "--volume", @config_with_roles.role(:web).cord_volume.docker_args[0]
+    assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}:/tmp/kamal-cord}, @config_with_roles.role(:web).cord_volume.docker_args[1]
+  end
+
+  test "cord container file" do
+    assert_equal "/tmp/kamal-cord/cord", @config_with_roles.role(:web).cord_container_file
+  end
 end

test/configuration/traefik/static_test.rb

@@ -1,26 +0,0 @@
-require "test_helper"
-
-class ConfigurationTraefikStaticTest < ActiveSupport::TestCase
-  setup do
-    @deploy = {
-      service: "app", image: "dhh/app",
-      registry: { "username" => "dhh", "password" => "secret" },
-      env: { "REDIS_URL" => "redis://x/y" },
-      servers: [ "1.1.1.1", "1.1.1.2" ],
-      volumes: ["/local/path:/container/path"],
-      traefik: {}
-    }
-
-    @config = Kamal::Configuration.new(@deploy)
-  end
-
-  test "env_file" do
-    ENV["EXAMPLE_API_KEY"] = "456"
-    @config.traefik["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
-    traefik_static = Kamal::Configuration::Traefik::Static.new(config: @config)
-
-    assert_equal "EXAMPLE_API_KEY=456\n", traefik_static.env_file
-  ensure
-    ENV.delete "EXAMPLE_API_KEY"
-  end
-end

test/configuration_test.rb

@@ -224,7 +224,7 @@ class ConfigurationTest < ActiveSupport::TestCase
         :volume_args=>["--volume", "/local/path:/container/path"],
         :builder=>{},
         :logging=>["--log-opt", "max-size=\"10m\""],
-        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7 }}
+        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }}
 
     assert_equal expected_config, @config.to_h
   end
@@ -253,9 +253,16 @@ class ConfigurationTest < ActiveSupport::TestCase
     assert_equal "/root/kamal", config.run_directory
   end
 
-  test "app traefik labels raise ArgumentError" do
+  test "run directory as docker volume" do
-    assert_raises(ArgumentError) do
+    config = Kamal::Configuration.new(@deploy)
-      Kamal::Configuration.new(@deploy.tap { |c| c.merge!(labels: { "traefik.enabled": true }) })
+    assert_equal "$(pwd)/.kamal", config.run_directory_as_docker_volume
-    end
+
+    config = Kamal::Configuration.new(@deploy.merge!(run_directory: "/root/kamal"))
+    assert_equal "/root/kamal", config.run_directory_as_docker_volume
+  end
+
+  test "run id" do
+    SecureRandom.expects(:hex).with(16).returns("09876543211234567890098765432112")
+    assert_equal "09876543211234567890098765432112", @config.run_id
   end
 end

test/integration/docker/deployer/Dockerfile

@@ -2,6 +2,8 @@ FROM ruby:3.2
 
 WORKDIR /app
 
+ENV VERBOSE=true
+
 RUN apt-get update --fix-missing && apt-get install -y ca-certificates openssh-client curl gnupg docker.io
 
 RUN install -m 0755 -d /etc/apt/keyrings

test/integration/docker/deployer/app/Dockerfile

@@ -4,4 +4,5 @@ COPY default.conf /etc/nginx/conf.d/default.conf
 
 ARG COMMIT_SHA
 RUN echo $COMMIT_SHA > /usr/share/nginx/html/version
+RUN mkdir -p /usr/share/nginx/html/versions && echo "version" > /usr/share/nginx/html/versions/$COMMIT_SHA
 

test/integration/docker/deployer/app/config/deploy.yml

@@ -8,6 +8,7 @@ env:
     CLEAR_TOKEN: '4321'
   secret:
     - SECRET_TOKEN
+asset_path: /usr/share/nginx/html/versions
 
 registry:
   server: registry:4443
@@ -18,8 +19,7 @@ builder:
   args:
     COMMIT_SHA: <%= `git rev-parse HEAD` %>
 healthcheck:
-  cmd: wget -qO- http://localhost > /dev/null
+  cmd: wget -qO- http://localhost > /dev/null || exit 1
-  path: /
 traefik:
   args:
     accesslog: true

test/integration/main_test.rb

@@ -20,6 +20,8 @@ class MainTest < IntegrationTest
     assert_app_is_up version: second_version
     assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy"
 
+    assert_accumulated_assets first_version, second_version
+
     kamal :rollback, first_version
     assert_hooks_ran "pre-connect", "pre-deploy", "post-deploy"
     assert_app_is_up version: first_version
@@ -39,12 +41,6 @@ class MainTest < IntegrationTest
     assert_no_remote_env_file
   end
 
-  test "envify" do
-    kamal :envify
-
-    assert_equal "SECRET_TOKEN=1234", deployer_exec("cat .env", capture: true)
-  end
-
   test "config" do
     config = YAML.load(kamal(:config, capture: true))
     version = latest_app_version
@@ -60,7 +56,7 @@ class MainTest < IntegrationTest
     assert_equal({ user: "root", auth_methods: [ "publickey" ], keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])
     assert_equal({ "multiarch" => false, "args" => { "COMMIT_SHA" => version } }, config[:builder])
     assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging]
-    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "cmd" => "wget -qO- http://localhost > /dev/null" }, config[:healthcheck])
+    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord"=>"/tmp/kamal-cord", "cmd"=>"wget -qO- http://localhost > /dev/null || exit 1" }, config[:healthcheck])
   end
 
   private
@@ -75,4 +71,10 @@ class MainTest < IntegrationTest
     def assert_no_remote_env_file
       assert_equal "nofile", docker_compose("exec vm1 stat /root/.kamal/env/roles/app-web.env 2> /dev/null || echo nofile", capture: true)
     end
+
+    def assert_accumulated_assets(*versions)
+      versions.each do |version|
+        assert_equal "200", Net::HTTP.get_response(URI.parse("http://localhost:12345/versions/#{version}")).code
+      end
+    end
 end