WIP

Merge pull request #439 from basecamp/zero-downtime-deploy-file
Zero downtime deployment with cord file
2023-09-08 16:40:41 +01:00 · 2023-09-07 09:34:40 +01:00 · 2023-09-07 09:34:22 +01:00 · 2023-09-06 14:35:30 +01:00 · 2023-09-06 14:33:13 +01:00 · 2023-09-06 14:31:07 +01:00
36 changed files with 589 additions and 596 deletions
--- a/bin/kamal
+++ b/bin/kamal
@@ -9,10 +9,10 @@ begin
  Kamal::Cli::Main.start(ARGV)
 rescue SSHKit::Runner::ExecuteError => e
  puts "  \e[31mERROR (#{e.cause.class}): #{e.message}\e[0m"
-  puts e.cause.backtrace
+  puts e.cause.backtrace if ENV["VERBOSE"]
  exit 1
 rescue => e
  puts "  \e[31mERROR (#{e.class}): #{e.message}\e[0m"
-  puts e.backtrace
+  puts e.backtrace if ENV["VERBOSE"]
  exit 1
 end
--- a/lib/kamal/cli.rb
+++ b/lib/kamal/cli.rb
@@ -1,7 +1,6 @@
 module Kamal::Cli
  class LockError < StandardError; end
  class HookError < StandardError; end
-  class TraefikError < StandardError; end
 end

 # SSHKit uses instance eval, so we need a global const for ergonomics
--- a/lib/kamal/cli/app.rb
+++ b/lib/kamal/cli/app.rb
@@ -2,8 +2,6 @@ class Kamal::Cli::App < Kamal::Cli::Base
  desc "boot", "Boot app on servers (or reboot app if already running)"
  def boot
    mutating do
-      ensure_traefik_file_provider_enabled
-
      hold_lock_on_error do
        say "Get most recent version available as an image...", :magenta unless options[:version]
        using_version(version_or_latest) do |version|
@@ -20,30 +18,44 @@ class Kamal::Cli::App < Kamal::Cli::Base
            roles.each do |role|
              app = KAMAL.app(role: role)
              auditor = KAMAL.auditor(role: role)
-              traefik_dynamic = KAMAL.traefik_dynamic(role: role)
              role_config = KAMAL.config.role(role)

-              if capture_with_info(*app.container_id_for_version(version, only_running: true), raise_on_non_zero_exit: false).present?
+              execute *app.extract_assets if role_config.assets?
+
+
+              if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
                tmp_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
                info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
                execute *auditor.record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
                execute *app.rename_container(version: version, new_version: tmp_version)
              end

+              old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+              original_old_version = old_version.gsub(/_replaced_[a-f0-9]{16}$/, "")
+
+              execute *app.sync_asset_volumes(old_version: original_old_version) if role_config.assets?
+
+              execute *app.tie_cord(role_config.cord_host_file) if role_config.uses_cord?
+
              execute *auditor.record("Booted app version #{version}"), verbosity: :debug

-              old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-              execute *app.start_or_run(hostname: "#{host}-#{SecureRandom.hex(6)}")
+              execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")

              Kamal::Utils::HealthcheckPoller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }

-              if role_config.running_traefik?
-                ip_address = capture_with_info(*app.ip_address(version: version)).strip
-                execute *traefik_dynamic.write_config(ip_address: ip_address)
-                Kamal::Utils::SwitchPoller.wait_for_switch(traefik_dynamic) { capture_with_info(*traefik_dynamic.run_id)&.strip }
-              end
+              if old_version.present?
+                if role_config.uses_cord?
+                  cord = capture_with_info(*app.cord(version: old_version), raise_on_non_zero_exit: false).strip
+                  if cord.present?
+                    execute *app.cut_cord(cord)
+                    Kamal::Utils::HealthcheckPoller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
+                  end
+                end

-              execute *app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
+                execute *app.stop(version: old_version), raise_on_non_zero_exit: false
+
+                execute *app.cleanup_assets if role_config.assets?
+              end
            end
          end
        end
@@ -54,23 +66,12 @@ class Kamal::Cli::App < Kamal::Cli::Base
  desc "start", "Start existing app container on servers"
  def start
    mutating do
-      ensure_traefik_file_provider_enabled
-
      on(KAMAL.hosts) do |host|
        roles = KAMAL.roles_on(host)

        roles.each do |role|
-          app = KAMAL.app(role: role)
-          role_config = KAMAL.config.role(role)
-
          execute *KAMAL.auditor.record("Started app version #{KAMAL.config.version}"), verbosity: :debug
-          execute *app.start, raise_on_non_zero_exit: false
-          version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-
-          if role_config.running_traefik?
-            ip_address = capture_with_info(*app.ip_address(version: version)).strip
-            execute *KAMAL.traefik_dynamic(role: role).write_config(ip_address: ip_address)
-          end
+          execute *KAMAL.app(role: role).start, raise_on_non_zero_exit: false
        end
      end
    end
@@ -83,10 +84,8 @@ class Kamal::Cli::App < Kamal::Cli::Base
        roles = KAMAL.roles_on(host)

        roles.each do |role|
-          app = KAMAL.app(role: role)
          execute *KAMAL.auditor.record("Stopped app", role: role), verbosity: :debug
-          execute *KAMAL.traefik_dynamic(role: role).remove_config if KAMAL.config.role(role).running_traefik?
-          execute *app.stop, raise_on_non_zero_exit: false
+          execute *KAMAL.app(role: role).stop, raise_on_non_zero_exit: false
        end
      end
    end
@@ -316,13 +315,4 @@ class Kamal::Cli::App < Kamal::Cli::Base
    def version_or_latest
      options[:version] || "latest"
    end
-
-    def ensure_traefik_file_provider_enabled
-      # Ensure traefik has been rebooted to switch to the file provider
-      on(KAMAL.traefik_hosts) do
-        unless capture_with_info(*KAMAL.traefik_static.docker_entrypoint_args).include?("--providers.file.directory=")
-          raise Kamal::Cli::TraefikError, "File provider not enabled, you'll need to run `kamal traefik reboot` to deploy"
-        end
-      end
-    end
 end
--- a/lib/kamal/cli/env.rb
+++ b/lib/kamal/cli/env.rb
@@ -13,9 +13,8 @@ class Kamal::Cli::Env < Kamal::Cli::Base
      end

      on(KAMAL.traefik_hosts) do
-        traefik_static_config = KAMAL.traefik_static.static_config
-        execute *KAMAL.traefik_static.make_env_directory
-        upload! StringIO.new(traefik_static_config.env_file), traefik_static_config.host_env_file_path, mode: 400
+        execute *KAMAL.traefik.make_env_directory
+        upload! StringIO.new(KAMAL.traefik.env_file), KAMAL.traefik.host_env_file_path, mode: 400
      end

      on(KAMAL.accessory_hosts) do
@@ -39,7 +38,7 @@ class Kamal::Cli::Env < Kamal::Cli::Base
      end

      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.traefik_static.remove_env_file
+        execute *KAMAL.traefik.remove_env_file
      end

      on(KAMAL.accessory_hosts) do
--- a/lib/kamal/cli/traefik.rb
+++ b/lib/kamal/cli/traefik.rb
@@ -4,8 +4,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
    mutating do
      on(KAMAL.traefik_hosts) do
        execute *KAMAL.registry.login
-        execute *KAMAL.traefik_static.ensure_config_directory
-        execute *KAMAL.traefik_static.start_or_run
+        execute *KAMAL.traefik.start_or_run
      end
    end
  end
@@ -17,10 +16,9 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
      on(KAMAL.traefik_hosts, in: options[:rolling] ? :sequence : :parallel) do
        execute *KAMAL.auditor.record("Rebooted traefik"), verbosity: :debug
        execute *KAMAL.registry.login
-        execute *KAMAL.traefik_static.stop
-        execute *KAMAL.traefik_static.remove_container
-        execute *KAMAL.traefik_static.ensure_config_directory
-        execute *KAMAL.traefik_static.run
+        execute *KAMAL.traefik.stop
+        execute *KAMAL.traefik.remove_container
+        execute *KAMAL.traefik.run
      end
    end
  end
@@ -30,7 +28,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
    mutating do
      on(KAMAL.traefik_hosts) do
        execute *KAMAL.auditor.record("Started traefik"), verbosity: :debug
-        execute *KAMAL.traefik_static.start
+        execute *KAMAL.traefik.start
      end
    end
  end
@@ -40,7 +38,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
    mutating do
      on(KAMAL.traefik_hosts) do
        execute *KAMAL.auditor.record("Stopped traefik"), verbosity: :debug
-        execute *KAMAL.traefik_static.stop
+        execute *KAMAL.traefik.stop
      end
    end
  end
@@ -55,7 +53,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base

  desc "details", "Show details about Traefik container from servers"
  def details
-    on(KAMAL.traefik_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.traefik_static.info), type: "Traefik" }
+    on(KAMAL.traefik_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.traefik.info), type: "Traefik" }
  end

  desc "logs", "Show log lines from Traefik on servers"
@@ -69,15 +67,15 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
    if options[:follow]
      run_locally do
        info "Following logs on #{KAMAL.primary_host}..."
-        info KAMAL.traefik_static.follow_logs(host: KAMAL.primary_host, grep: grep)
-        exec KAMAL.traefik_static.follow_logs(host: KAMAL.primary_host, grep: grep)
+        info KAMAL.traefik.follow_logs(host: KAMAL.primary_host, grep: grep)
+        exec KAMAL.traefik.follow_logs(host: KAMAL.primary_host, grep: grep)
      end
    else
      since = options[:since]
      lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set

      on(KAMAL.traefik_hosts) do |host|
-        puts_by_host host, capture(*KAMAL.traefik_static.logs(since: since, lines: lines, grep: grep)), type: "Traefik"
+        puts_by_host host, capture(*KAMAL.traefik.logs(since: since, lines: lines, grep: grep)), type: "Traefik"
      end
    end
  end
@@ -96,7 +94,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
    mutating do
      on(KAMAL.traefik_hosts) do
        execute *KAMAL.auditor.record("Removed traefik container"), verbosity: :debug
-        execute *KAMAL.traefik_static.remove_container
+        execute *KAMAL.traefik.remove_container
      end
    end
  end
@@ -106,7 +104,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
    mutating do
      on(KAMAL.traefik_hosts) do
        execute *KAMAL.auditor.record("Removed traefik image"), verbosity: :debug
-        execute *KAMAL.traefik_static.remove_image
+        execute *KAMAL.traefik.remove_image
      end
    end
  end
--- a/lib/kamal/commander.rb
+++ b/lib/kamal/commander.rb
@@ -81,7 +81,7 @@ class Kamal::Commander


  def app(role: nil)
-    Kamal::Commands::App.new(config, role: role || config.roles.first.name)
+    Kamal::Commands::App.new(config, role: role)
  end

  def accessory(name)
@@ -124,12 +124,8 @@ class Kamal::Commander
    @server ||= Kamal::Commands::Server.new(config)
  end

-  def traefik_static
-    @traefik_static ||= Kamal::Commands::Traefik::Static.new(config)
-  end
-
-  def traefik_dynamic(role: nil)
-    Kamal::Commands::Traefik::Dynamic.new(config, role: role || config.roles.first.name)
+  def traefik
+    @traefik ||= Kamal::Commands::Traefik.new(config)
  end

  def with_verbosity(level)
--- a/lib/kamal/commands/accessory.rb
+++ b/lib/kamal/commands/accessory.rb
@@ -86,10 +86,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
    end
  end

-  def make_directory_for(remote_file)
-    make_directory Pathname.new(remote_file).dirname.to_s
-  end
-
  def remove_service_directory
    [ :rm, "-rf", service_name ]
  end
--- a/lib/kamal/commands/app.rb
+++ b/lib/kamal/commands/app.rb
@@ -1,15 +1,12 @@
 class Kamal::Commands::App < Kamal::Commands::Base
  ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]

-  attr_reader :role
+  attr_reader :role, :role_config

  def initialize(config, role: nil)
    super(config)
    @role = role
-  end
-
-  def start_or_run(hostname: nil)
-    combine start, run(hostname: hostname), by: "||"
+    @role_config = config.role(self.role)
  end

  def run(hostname: nil)
@@ -23,6 +20,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
      *role_config.health_check_args,
      *config.logging_args,
      *config.volume_args,
+      *role_config.asset_volume_args,
      *role_config.label_args,
      *role_config.option_args,
      config.absolute_image,
@@ -47,10 +45,6 @@ class Kamal::Commands::App < Kamal::Commands::Base
    docker :ps, *filter_args
  end

-  def ip_address(version:)
-    docker :inspect, "-f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}'", container_name(version)
-  end
-

  def logs(since: nil, lines: nil, grep: nil)
    pipe \
@@ -112,7 +106,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
  def list_versions(*docker_args, statuses: nil)
    pipe \
      docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
-      %(while read line; do echo ${line##{role_config.full_name}-}; done) # Extract SHA from "service-role-dest-SHA"
+      %(while read line; do echo ${line##{role_config.container_prefix}-}; done) # Extract SHA from "service-role-dest-SHA"
  end

  def list_containers
@@ -150,26 +144,74 @@ class Kamal::Commands::App < Kamal::Commands::Base
  end

  def make_env_directory
-    make_directory config.role(role).host_env_directory
+    make_directory role_config.host_env_directory
  end

  def remove_env_file
-    [:rm, "-f", config.role(role).host_env_file_path]
+    [:rm, "-f", role_config.host_env_file_path]
  end

-  def service_role_dest
-    [config.service, role, config.destination].compact.join("-")
+  def cord(version:)
+    pipe \
+      docker(:inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", container_name(version)),
+      [:awk, "'$2 == \"#{role_config.cord_volume.container_path}\" {print $1}'"]
+  end
+
+  def tie_cord(cord)
+    create_empty_file(cord)
+  end
+
+  def cut_cord(cord)
+    remove_directory(cord)
+  end
+
+  def extract_assets
+    asset_container = "#{role_config.container_prefix}-assets"
+
+    combine \
+      make_directory(role_config.asset_extracted_path),
+      [*docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true"],
+      docker(:run, "--name", asset_container, "--detach", "--rm", config.latest_image, "sleep infinity"),
+      docker(:cp, "-L", "#{asset_container}:#{role_config.asset_path}/.", role_config.asset_extracted_path),
+      docker(:stop, "-t 1", asset_container),
+      by: "&&"
+  end
+
+  def sync_asset_volumes(old_version: nil)
+    new_extracted_path, new_volume_path = role_config.asset_extracted_path(config.version), role_config.asset_volume.host_path
+    if old_version.present?
+      old_extracted_path, old_volume_path = role_config.asset_extracted_path(old_version), role_config.asset_volume(old_version).host_path
+    end
+
+    commands = [make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path)]
+
+    if old_version.present?
+      commands << copy_contents(new_extracted_path, old_volume_path)
+      commands << copy_contents(old_extracted_path, new_volume_path)
+    end
+
+    chain *commands
+  end
+
+  def cleanup_assets
+    chain \
+      find_and_remove_older_siblings(role_config.asset_extracted_path),
+      find_and_remove_older_siblings(role_config.asset_volume_path)
  end

  private
    def container_name(version = nil)
-      [ role_config.full_name, version || config.version ].compact.join("-")
+      [ role_config.container_prefix, version || config.version ].compact.join("-")
    end

    def filter_args(statuses: nil)
      argumentize "--filter", filters(statuses: statuses)
    end

+    def service_role_dest
+      [config.service, role, config.destination].compact.join("-")
+    end
+
    def filters(statuses: nil)
      [ "label=service=#{config.service}" ].tap do |filters|
        filters << "label=destination=#{config.destination}" if config.destination
@@ -180,7 +222,18 @@ class Kamal::Commands::App < Kamal::Commands::Base
      end
    end

-    def role_config
-      @role_config ||= config.role(self.role)
+    def find_and_remove_older_siblings(path)
+      [
+        :find,
+        Pathname.new(path).dirname,
+        "-maxdepth 1",
+        "-name", "'#{role_config.container_prefix}-*'",
+        "!", "-name", Pathname.new(path).basename,
+        "-exec rm -rf \"{}\" +"
+      ]
+    end
+
+    def copy_contents(source, destination)
+      [ :cp, "-rn", "#{source}/*", destination ]
    end
 end
--- a/lib/kamal/commands/base.rb
+++ b/lib/kamal/commands/base.rb
@@ -34,6 +34,10 @@ module Kamal::Commands
      [ :mkdir, "-p", path ]
    end

+    def remove_directory(path)
+      [ :rm, "-r", path ]
+    end
+
    private
      def combine(*commands, by: "&&")
        commands
@@ -69,5 +73,11 @@ module Kamal::Commands
      def tags(**details)
        Kamal::Tags.from_config(config, **details)
      end
+
+      def create_empty_file(file)
+        chain \
+          make_directory_for(file),
+          [:touch, file]
+      end
  end
 end
--- a/lib/kamal/commands/healthcheck.rb
+++ b/lib/kamal/commands/healthcheck.rb
@@ -1,5 +1,4 @@
 class Kamal::Commands::Healthcheck < Kamal::Commands::Base
-  EXPOSED_PORT = 3999

  def run
    web = config.role(:web)
@@ -7,11 +6,11 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
    docker :run,
      "--detach",
      "--name", container_name_with_version,
-      "--publish", "#{EXPOSED_PORT}:#{config.healthcheck["port"]}",
+      "--publish", "#{exposed_port}:#{config.healthcheck["port"]}",
      "--label", "service=#{container_name}",
      "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
      *web.env_args,
-      *web.health_check_args,
+      *web.health_check_args(cord: false),
      *config.volume_args,
      *web.option_args,
      config.absolute_image,
@@ -52,6 +51,10 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
    end

    def health_url
-      "http://localhost:#{EXPOSED_PORT}#{config.healthcheck["path"]}"
+      "http://localhost:#{exposed_port}#{config.healthcheck["path"]}"
+    end
+
+    def exposed_port
+      config.healthcheck["exposed_port"]
    end
 end
--- a/lib/kamal/commands/server.rb
+++ b/lib/kamal/commands/server.rb
@@ -1,5 +1,5 @@
 class Kamal::Commands::Server < Kamal::Commands::Base
  def ensure_run_directory
-    make_directory config.run_directory
+    [:mkdir, "-p", config.run_directory]
  end
 end
--- a/lib/kamal/commands/traefik.rb
+++ b/lib/kamal/commands/traefik.rb
@@ -0,0 +1,122 @@
+class Kamal::Commands::Traefik < Kamal::Commands::Base
+  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
+
+  DEFAULT_IMAGE = "traefik:v2.9"
+  CONTAINER_PORT = 80
+  DEFAULT_ARGS = {
+    'log.level' => 'DEBUG'
+  }
+
+  def run
+    docker :run, "--name traefik",
+      "--detach",
+      "--restart", "unless-stopped",
+      *publish_args,
+      "--volume", "/var/run/docker.sock:/var/run/docker.sock",
+      *env_args,
+      *config.logging_args,
+      *label_args,
+      *docker_options_args,
+      image,
+      "--providers.docker",
+      *cmd_option_args
+  end
+
+  def start
+    docker :container, :start, "traefik"
+  end
+
+  def stop
+    docker :container, :stop, "traefik"
+  end
+
+  def start_or_run
+    combine start, run, by: "||"
+  end
+
+  def info
+    docker :ps, "--filter", "name=^traefik$"
+  end
+
+  def logs(since: nil, lines: nil, grep: nil)
+    pipe \
+      docker(:logs, "traefik", (" --since #{since}" if since), (" --tail #{lines}" if lines), "--timestamps", "2>&1"),
+      ("grep '#{grep}'" if grep)
+  end
+
+  def follow_logs(host:, grep: nil)
+    run_over_ssh pipe(
+      docker(:logs, "traefik", "--timestamps", "--tail", "10", "--follow", "2>&1"),
+      (%(grep "#{grep}") if grep)
+    ).join(" "), host: host
+  end
+
+  def remove_container
+    docker :container, :prune, "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
+  end
+
+  def remove_image
+    docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
+  end
+
+  def port
+    "#{host_port}:#{CONTAINER_PORT}"
+  end
+
+  def env_file
+    env_file_with_secrets config.traefik.fetch("env", {})
+  end
+
+  def host_env_file_path
+    File.join host_env_directory, "traefik.env"
+  end
+
+  def make_env_directory
+    make_directory(host_env_directory)
+  end
+
+  def remove_env_file
+    [:rm, "-f", host_env_file_path]
+  end
+
+  private
+    def publish_args
+      argumentize "--publish", port unless config.traefik["publish"] == false
+    end
+
+    def label_args
+      argumentize "--label", labels
+    end
+
+    def env_args
+      argumentize "--env-file", host_env_file_path
+    end
+
+    def host_env_directory
+      File.join config.host_env_directory, "traefik"
+    end
+
+    def labels
+      config.traefik["labels"] || []
+    end
+
+    def image
+      config.traefik.fetch("image") { DEFAULT_IMAGE }
+    end
+
+    def docker_options_args
+      optionize(config.traefik["options"] || {})
+    end
+
+    def cmd_option_args
+      if args = config.traefik["args"]
+        optionize DEFAULT_ARGS.merge(args), with: "="
+      else
+        optionize DEFAULT_ARGS, with: "="
+      end
+    end
+
+    def host_port
+      config.traefik["host_port"] || CONTAINER_PORT
+    end
+end
--- a/lib/kamal/commands/traefik/dynamic.rb
+++ b/lib/kamal/commands/traefik/dynamic.rb
@@ -1,43 +0,0 @@
-class Kamal::Commands::Traefik::Dynamic < Kamal::Commands::Base
-  attr_reader :static_config, :dynamic_config
-
-  def initialize(config, role: nil)
-    super(config)
-    @static_config = Kamal::Configuration::Traefik::Static.new(config: config)
-    @dynamic_config = Kamal::Configuration::Traefik::Dynamic.new(config: config, role: role)
-  end
-
-  def run_id
-    pipe \
-      [:docker, :exec, :traefik, :wget, "-qSO", "/dev/null", "http://localhost:#{Kamal::Configuration::Traefik::Static::CONTAINER_PORT}#{config.healthcheck["path"]}", "2>&1"],
-      [:grep, "-i", Kamal::Configuration::Traefik::Dynamic::RUN_ID_HEADER],
-      [:cut, "-d ' ' -f 4"]
-  end
-
-  def write_config(ip_address:)
-    # Write to tmp then mv for an atomic copy. If you write directly traefik sees an empty file
-    # and removes the service before picking up the new config.
-    temp_config_file = "/tmp/kamal-traefik-config-#{rand(10000000)}"
-    chain \
-      write([:echo, dynamic_config.config(ip_address: ip_address).to_yaml.shellescape], temp_config_file),
-      [:mv, temp_config_file, host_file]
-  end
-
-  def remove_config
-    [:rm, host_file]
-  end
-
-  def boot_check?
-    dynamic_config.boot_check?
-  end
-
-  def config_run_id
-    dynamic_config.run_id
-  end
-
-  private
-    def host_file
-      "#{static_config.host_directory}/#{dynamic_config.host_file}"
-    end
-end
-
--- a/lib/kamal/commands/traefik/static.rb
+++ b/lib/kamal/commands/traefik/static.rb
@@ -1,70 +0,0 @@
-class Kamal::Commands::Traefik::Static < Kamal::Commands::Base
-  attr_reader :static_config, :dynamic_config
-
-  def initialize(config, role: nil)
-    super(config)
-    @static_config = Kamal::Configuration::Traefik::Static.new(config: config)
-  end
-
-  def run
-    docker :run, static_config.docker_args, static_config.image, static_config.traefik_args
-  end
-
-  def start
-    docker :container, :start, "traefik"
-  end
-
-  def stop
-    docker :container, :stop, "traefik"
-  end
-
-  def start_or_run
-    combine start, run, by: "||"
-  end
-
-  def info
-    docker :ps, "--filter", "name=^traefik$"
-  end
-
-  def logs(since: nil, lines: nil, grep: nil)
-    pipe \
-      docker(:logs, "traefik", (" --since #{since}" if since), (" --tail #{lines}" if lines), "--timestamps", "2>&1"),
-      ("grep '#{grep}'" if grep)
-  end
-
-  def follow_logs(host:, grep: nil)
-    run_over_ssh pipe(
-      docker(:logs, "traefik", "--timestamps", "--tail", "10", "--follow", "2>&1"),
-      (%(grep "#{grep}") if grep)
-    ).join(" "), host: host
-  end
-
-  def remove_container
-    docker :container, :prune, "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
-  end
-
-  def remove_image
-    docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
-  end
-
-  def port
-    "#{host_port}:#{CONTAINER_PORT}"
-  end
-
-  def make_env_directory
-    make_directory(static_config.host_env_directory)
-  end
-
-  def remove_env_file
-    [:rm, "-f", static_config.host_env_file_path]
-  end
-
-  def ensure_config_directory
-    make_directory(static_config.host_directory)
-  end
-
-  def docker_entrypoint_args
-    docker :inspect, "-f '{{index .Args 1 }}'", :traefik
-  end
-end
-
--- a/lib/kamal/configuration.rb
+++ b/lib/kamal/configuration.rb
@@ -7,7 +7,7 @@ require "net/ssh/proxy/jump"

 class Kamal::Configuration
  delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils

  attr_accessor :destination
  attr_accessor :raw_config
@@ -61,6 +61,14 @@ class Kamal::Configuration
    raw_config.run_directory || ".kamal"
  end

+  def run_directory_as_docker_volume
+    if Pathname.new(run_directory).absolute?
+      run_directory
+    else
+      File.join "$(pwd)", run_directory
+    end
+  end
+

  def roles
    @roles ||= role_names.collect { |role_name| Role.new(role_name, config: self) }
@@ -141,7 +149,7 @@ class Kamal::Configuration


  def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7 }.merge(raw_config.healthcheck || {})
+    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }.merge(raw_config.healthcheck || {})
  end

  def readiness_delay
@@ -153,7 +161,7 @@ class Kamal::Configuration
  end

  def valid?
-    ensure_required_keys_present && ensure_valid_kamal_version && ensure_no_traefik_labels
+    ensure_required_keys_present && ensure_valid_kamal_version
  end


@@ -199,6 +207,14 @@ class Kamal::Configuration
    "#{run_directory}/env"
  end

+  def run_id
+    @run_id ||= SecureRandom.hex(16)
+  end
+
+  def asset_path
+    raw_config.asset_path
+  end
+
  private
    # Will raise ArgumentError if any required config keys are missing
    def ensure_required_keys_present
@@ -231,17 +247,6 @@ class Kamal::Configuration
      true
    end

-    def ensure_no_traefik_labels
-      # The switch to a traefik file provider means that traefik labels on app containers are ignored
-      # We'll raise an error and suggest moving them
-
-      if roles.any? { |role| role.labels.keys.any? { |label| label.start_with?("traefik.") } }
-        raise ArgumentError, "Traefik is not configured to read labels, move traefik config to dynamic:"
-      end
-
-      true
-    end
-

    def role_names
      raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort
--- a/lib/kamal/configuration/role.rb
+++ b/lib/kamal/configuration/role.rb
@@ -1,10 +1,11 @@
 class Kamal::Configuration::Role
+  CORD_FILE = "cord"
  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils

  attr_accessor :name

  def initialize(name, config:)
-    @name, @config = name.inquiry, config
+   @name, @config = name.inquiry, config
  end

  def primary_host
@@ -16,7 +17,7 @@ class Kamal::Configuration::Role
  end

  def labels
-    default_labels.merge(custom_labels)
+    default_labels.merge(traefik_labels).merge(custom_labels)
  end

  def label_args
@@ -47,28 +48,64 @@ class Kamal::Configuration::Role
    argumentize "--env-file", host_env_file_path
  end

-  def health_check_args
+  def asset_volume_args
+    asset_volume&.docker_args
+  end
+
+  def health_check_args(cord: true)
    if health_check_cmd.present?
-      optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval })
+      if cord && uses_cord?
+        optionize({ "health-cmd" => health_check_cmd_with_cord, "health-interval" => health_check_interval })
+          .concat(cord_volume.docker_args)
+      else
+        optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval })
+      end
    else
      []
    end
  end

  def health_check_cmd
-    options = specializations["healthcheck"] || {}
-    options = config.healthcheck.merge(options) if running_traefik?
+    health_check_options["cmd"] || http_health_check(port: health_check_options["port"], path: health_check_options["path"])
+  end

-    options["cmd"] || http_health_check(port: options["port"], path: options["path"])
+  def health_check_cmd_with_cord
+    "(#{health_check_cmd}) && (stat #{cord_container_file} > /dev/null || exit 1)"
  end

  def health_check_interval
-    options = specializations["healthcheck"] || {}
-    options = config.healthcheck.merge(options) if running_traefik?
-
-    options["interval"] || "1s"
+    health_check_options["interval"] || "1s"
  end

+  def uses_cord?
+    running_traefik? && cord_volume && health_check_cmd.present?
+  end
+
+  def cord_host_directory
+    File.join config.run_directory_as_docker_volume, "cords", [container_prefix, config.run_id].join("-")
+  end
+
+  def cord_volume
+    if (cord = health_check_options["cord"])
+      @cord_volume ||= Kamal::Configuration::Volume.new \
+        host_path: File.join(config.run_directory, "cords", [container_prefix, config.run_id].join("-")),
+        container_path: cord
+    end
+  end
+
+  def cord_host_file
+    File.join cord_volume.host_path, CORD_FILE
+  end
+
+  def cord_container_directory
+    health_check_options.fetch("cord", nil)
+  end
+
+  def cord_container_file
+    File.join cord_volume.container_path, CORD_FILE
+  end
+
+
  def cmd
    specializations["cmd"]
  end
@@ -82,22 +119,39 @@ class Kamal::Configuration::Role
  end

  def running_traefik?
-    name.web? || (specializations["traefik"] != nil && specializations["traefik"] != false)
+    name.web? || specializations["traefik"]
  end

-  def traefik
-    case specializations["traefik"]
-    when NilClass, TrueClass, FalseClass
-      {}
-    else
-      specializations["traefik"]
-    end
+  def container_name(version = nil)
+    [ container_prefix, version || config.version ].compact.join("-")
  end

-  def full_name
+  def container_prefix
    [ config.service, name, config.destination ].compact.join("-")
  end

+  def asset_path
+    specializations["asset_path"] || config.asset_path
+  end
+
+  def assets?
+    asset_path.present? && running_traefik?
+  end
+
+  def asset_volume(version = nil)
+    if assets?
+      Kamal::Configuration::Volume.new \
+        host_path: asset_volume_path(version), container_path: asset_path
+    end
+  end
+
+  def asset_extracted_path(version = nil)
+    File.join config.run_directory, "assets", "extracted", container_name(version)
+  end
+
+  def asset_volume_path(version = nil)
+    File.join config.run_directory, "assets", "volumes", container_name(version)
+  end

  private
    attr_accessor :config
@@ -119,6 +173,26 @@ class Kamal::Configuration::Role
      end
    end

+    def traefik_labels
+      if running_traefik?
+        {
+          # Setting a service property ensures that the generated service name will be consistent between versions
+          "traefik.http.services.#{traefik_service}.loadbalancer.server.scheme" => "http",
+
+          "traefik.http.routers.#{traefik_service}.rule" => "PathPrefix(`/`)",
+          "traefik.http.middlewares.#{traefik_service}-retry.retry.attempts" => "5",
+          "traefik.http.middlewares.#{traefik_service}-retry.retry.initialinterval" => "500ms",
+          "traefik.http.routers.#{traefik_service}.middlewares" => "#{traefik_service}-retry@docker"
+        }
+      else
+        {}
+      end
+    end
+
+    def traefik_service
+      [ config.service, name, config.destination ].compact.join("-")
+    end
+
    def custom_labels
      Hash.new.tap do |labels|
        labels.merge!(config.labels) if config.labels.present?
@@ -158,4 +232,12 @@ class Kamal::Configuration::Role
    def http_health_check(port:, path:)
      "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present?
    end
+
+    def health_check_options
+      @health_check_options ||= begin
+        options = specializations["healthcheck"] || {}
+        options = config.healthcheck.merge(options) if running_traefik?
+        options
+      end
+    end
 end
--- a/lib/kamal/configuration/traefik/dynamic.rb
+++ b/lib/kamal/configuration/traefik/dynamic.rb
@@ -1,66 +0,0 @@
-class Kamal::Configuration::Traefik::Dynamic
-  RUN_ID_HEADER = "X-Kamal-Run-ID"
-
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
-
-  attr_reader :traefik_config, :role_config, :role_traefik_config
-
-  def initialize(config:, role:)
-    @traefik_config = config.traefik || {}
-    @role_config = config.role(role)
-    @role_traefik_config = role_config&.traefik || {}
-  end
-
-  def host_file
-    "#{role_config.full_name}.yml"
-  end
-
-  def config(ip_address:)
-    default_config(ip_address:).deep_merge!(custom_config)
-  end
-
-  def boot_check?
-    role_traefik_config.fetch("boot_check") { traefik_config.fetch("boot_check", true) }
-  end
-
-  def run_id
-    @run_id ||= SecureRandom.hex(16)
-  end
-
-  private
-    def default_config(ip_address:)
-      run_id_header_middleware = "#{role_config.full_name}-id-header"
-
-      {
-        "http" => {
-          "routers" => {
-            role_config.full_name => {
-              "rule" => "PathPrefix(`/`)",
-              "middlewares" => [ run_id_header_middleware ],
-              "service" => role_config.full_name
-            }
-          },
-          "services" => {
-            role_config.full_name => {
-              "loadbalancer" => {
-                "servers" => [ { "url" => "http://#{ip_address}:80" } ]
-              }
-            }
-          },
-          "middlewares" => {
-            run_id_header_middleware => {
-              "headers" => {
-                "customresponseheaders" => {
-                  RUN_ID_HEADER => run_id
-                }
-              }
-            }
-          }
-        }
-      }
-    end
-
-    def custom_config
-      traefik_config.fetch("dynamic", {}).deep_merge(role_traefik_config.fetch("dynamic", {}))
-    end
-end
--- a/lib/kamal/configuration/traefik/static.rb
+++ b/lib/kamal/configuration/traefik/static.rb
@@ -1,84 +0,0 @@
-class Kamal::Configuration::Traefik::Static
-  CONTAINER_PORT = 80
-  DEFAULT_IMAGE = "traefik:v2.9"
-  CONFIG_DIRECTORY = "/var/run/traefik-config"
-  DEFAULT_ARGS = {
-    "providers.docker": true, # Obsolete now but required for zero-downtime upgrade from previous versions
-    "providers.file.directory" =>  "/var/run/traefik-config",
-    "providers.file.watch": true,
-    "log.level" => "DEBUG",
-  }
-
-  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
-
-  attr_reader :config, :traefik_config
-
-  def initialize(config:)
-    @config = config
-    @traefik_config = config.traefik || {}
-  end
-
-  def docker_args
-    [
-      "--name traefik",
-      "--detach",
-      "--restart", "unless-stopped",
-      *publish_args,
-      "--volume", "/var/run/docker.sock:/var/run/docker.sock",
-      "--volume", "#{host_directory}:#{CONFIG_DIRECTORY}",
-      *env_args,
-      *config.logging_args,
-      *label_args,
-      *docker_options_args
-    ]
-  end
-
-  def image
-    traefik_config.fetch("image") { DEFAULT_IMAGE }
-  end
-
-  def traefik_args
-    optionize DEFAULT_ARGS.merge(traefik_config.fetch("args", {})), with: "="
-  end
-
-  def host_directory
-    if Pathname.new(config.run_directory).absolute?
-      "#{config.run_directory}/traefik-config"
-    else
-      "$(pwd)/#{config.run_directory}/traefik-config"
-    end
-  end
-
-  def host_env_file_path
-    File.join host_env_directory, "traefik.env"
-  end
-
-  def host_env_directory
-    File.join config.host_env_directory, "traefik"
-  end
-
-  def env_file
-    env_file_with_secrets config.traefik.fetch("env", {})
-  end
-
-  private
-    def host_port
-      traefik_config.fetch("host_port", CONTAINER_PORT)
-    end
-
-    def publish_args
-      argumentize "--publish", "#{host_port}:#{CONTAINER_PORT}" unless traefik_config["publish"] == false
-    end
-
-    def env_args
-      argumentize "--env-file", host_env_file_path
-    end
-
-    def label_args
-      argumentize "--label", traefik_config.fetch("labels", [])
-    end
-
-    def docker_options_args
-      optionize(traefik_config["options"] || {})
-    end
-end
--- a/lib/kamal/configuration/volume.rb
+++ b/lib/kamal/configuration/volume.rb
@@ -0,0 +1,22 @@
+class Kamal::Configuration::Volume
+  attr_reader :host_path, :container_path
+  delegate :argumentize, to: Kamal::Utils
+
+  def initialize(host_path:, container_path:)
+    @host_path = host_path
+    @container_path = container_path
+  end
+
+  def docker_args
+    argumentize "--volume", "#{host_path_for_docker_volume}:#{container_path}"
+  end
+
+  private
+    def host_path_for_docker_volume
+      if Pathname.new(host_path).absolute?
+        host_path
+      else
+        File.join "$(pwd)", host_path
+      end
+    end
+end
--- a/lib/kamal/utils.rb
+++ b/lib/kamal/utils.rb
@@ -115,25 +115,4 @@ module Kamal::Utils

    "#{key.to_s}=#{value.to_s}\n"
  end
-
-  def poll(max_attempts:, exception:, &block)
-    attempt = 1
-
-    begin
-      block.call
-    rescue exception => e
-      if attempt <= max_attempts
-        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-        sleep attempt
-        attempt += 1
-        retry
-      else
-        raise
-      end
-    end
-  end
-
-  def info(message)
-    SSHKit.config.output.info(message)
-  end
 end
--- a/lib/kamal/utils/healthcheck_poller.rb
+++ b/lib/kamal/utils/healthcheck_poller.rb
@@ -1,22 +1,64 @@
 class Kamal::Utils::HealthcheckPoller
-  TRAEFIK_HEALTHY_DELAY = 2
+  TRAEFIK_UPDATE_DELAY = 2

  class HealthcheckError < StandardError; end

  class << self
    def wait_for_healthy(pause_after_ready: false, &block)
-      Kamal::Utils.poll(max_attempts: KAMAL.config.healthcheck["max_attempts"], exception: HealthcheckError) do
+      attempt = 1
+      max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+      begin
        case status = block.call
        when "healthy"
-          sleep TRAEFIK_HEALTHY_DELAY if pause_after_ready
+          sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
        when "running" # No health check configured
          sleep KAMAL.config.readiness_delay if pause_after_ready
        else
          raise HealthcheckError, "container not ready (#{status})"
        end
-
-        SSHKit.config.output.info "Container is healthy!"
+      rescue HealthcheckError => e
+        if attempt <= max_attempts
+          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+          sleep attempt
+          attempt += 1
+          retry
+        else
+          raise
+        end
      end
+
+      info "Container is healthy!"
    end
+
+    def wait_for_unhealthy(pause_after_ready: false, &block)
+      attempt = 1
+      max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+      begin
+        case status = block.call
+        when "unhealthy"
+          sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
+        else
+          raise HealthcheckError, "container not unhealthy (#{status})"
+        end
+      rescue HealthcheckError => e
+        if attempt <= max_attempts
+          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+          sleep attempt
+          attempt += 1
+          retry
+        else
+          raise
+        end
+      end
+
+      info "Container is unhealthy!"
+    end
+
+    private
+      def info(message)
+        SSHKit.config.output.info(message)
+      end
  end
 end
--- a/lib/kamal/utils/switch_poller.rb
+++ b/lib/kamal/utils/switch_poller.rb
@@ -1,17 +0,0 @@
-class Kamal::Utils::SwitchPoller
-  class SwitchError < StandardError; end
-
-  class << self
-    TRAEFIK_SWITCH_DELAY = 2
-    def wait_for_switch(traefik_dynamic, &block)
-      if traefik_dynamic.boot_check?
-        Kamal::Utils.poll(max_attempts: 5, exception: SwitchError) do
-          polled_run_id = block.call
-          raise SwitchError, "Waiting for #{traefik_dynamic.config_run_id}, currently #{polled_run_id}" unless polled_run_id == traefik_dynamic.config_run_id
-        end
-      else
-        sleep TRAEFIK_SWITCH_DELAY
-      end
-    end
-  end
-end
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -5,21 +5,43 @@ class CliAppTest < CliTestCase
    stub_running
    run_command("boot").tap do |output|
      assert_match "docker tag dhh/app:latest dhh/app:latest", output
-      assert_match /docker rename app-web-latest app-web-latest_replaced_[0-9a-f]{16}/, output
      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} /, output
      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
    end
  end

  test "boot will rename if same version is already running" do
+    Object.any_instance.stubs(:sleep)
    run_command("details") # Preheat Kamal const

-    stub_running
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678") # running version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .returns("cordfile") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("unhealthy") # old version unhealthy
+
    run_command("boot").tap do |output|
      assert_match /Renaming container .* to .* as already deployed on 1.1.1.1/, output # Rename
+      assert_match /docker rename app-web-latest app-web-latest_replaced_[0-9a-f]{16}/, output
      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} /, output
      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
    end
+  ensure
+    Thread.report_on_exception = true
  end

  test "boot uses group strategy when specified" do
@@ -32,27 +54,9 @@ class CliAppTest < CliTestCase
    run_command("boot", config: :with_boot_strategy)
  end

-  test "boot without traefik file provider raises exception" do
-    Thread.report_on_exception = false
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{index .Args 1 }}'", :traefik)
-      .returns("[--providers.docker --log.level=DEBUG --accesslog --accesslog.format=json]").at_least_once
-
-    assert_raises(SSHKit::Runner::ExecuteError, "Exception while executing on host 1.1.1.1: File provider not enabled, you'll need to run `kamal traefik reboot` to deploy") do
-      run_command("boot")
-    end
-  ensure
-    Thread.report_on_exception = true
-  end
-
  test "boot errors leave lock in place" do
    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }

-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{index .Args 1 }}'", :traefik)
-      .returns("[--providers.docker --providers.file.directory=/var/run/traefik-config --providers.file.watch --log.level=DEBUG --accesslog --accesslog.format=json]").at_least_once
-
    Kamal::Cli::App.any_instance.expects(:using_version).raises(RuntimeError)

    assert !KAMAL.holding_lock?
@@ -63,13 +67,6 @@ class CliAppTest < CliTestCase
  end

  test "start" do
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with { |*args| args == [ :docker, :inspect, "-f '{{index .Args 1 }}'", :traefik ] }
-      .returns("[--providers.docker --providers.file.directory=/var/run/traefik-config --providers.file.watch --log.level=DEBUG --accesslog --accesslog.format=json]").at_least_once
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with { |*args| args != [ :docker, :inspect, "-f '{{index .Args 1 }}'", :traefik ] }
-      .returns("").at_least_once
-
    run_command("start").tap do |output|
      assert_match "docker start app-web-999", output
    end
@@ -136,7 +133,7 @@ class CliAppTest < CliTestCase

  test "exec" do
    run_command("exec", "ruby -v").tap do |output|
-      assert_match "docker run --rm --env-file .kamal/env/roles/app-web.env dhh/app:latest ruby -v", output
+      assert_match "docker run --rm dhh/app:latest ruby -v", output
    end
  end

@@ -192,26 +189,16 @@ class CliAppTest < CliTestCase
    end

    def stub_running
-      SecureRandom.stubs(:hex).with(16).returns("12345678901234567890123456789012")
-      SecureRandom.stubs(:hex).with(6).returns("123456789012")
-      SecureRandom.stubs(:hex).with(8).returns("1234567890123456")
+      Object.any_instance.stubs(:sleep)

      SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version

-      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :inspect, "-f '{{index .Args 1 }}'", :traefik)
-        .returns("[--providers.docker --providers.file.directory=/var/run/traefik-config --providers.file.watch --log.level=DEBUG --accesslog --accesslog.format=json]").at_least_once
-
      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
        .returns("running") # health check

      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :inspect, "-f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}'", "app-web-latest")
-        .returns("172.17.0.3").at_least_once
-
-      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :exec, :traefik, :wget, "-qSO", "/dev/null", "http://localhost:80/up", "2>&1", "|", :grep, "-i", "X-Kamal-Run-ID", "|", :cut, "-d ' ' -f 4")
-        .returns("12345678901234567890123456789012").at_least_once
+        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+        .returns("unhealthy") # health check
    end
 end
--- a/test/cli/healthcheck_test.rb
+++ b/test/cli/healthcheck_test.rb
@@ -5,7 +5,8 @@ class CliHealthcheckTest < CliTestCase
    # Prevent expected failures from outputting to terminal
    Thread.report_on_exception = false

-    Object.any_instance.stubs(:sleep) # No sleeping when retrying
+    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")

    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
@@ -28,15 +29,13 @@ class CliHealthcheckTest < CliTestCase
      assert_match "container not ready (unhealthy), retrying in 2s (attempt 2/7)...", output
      assert_match "Container is healthy!", output
    end
-  ensure
-    Thread.report_on_exception = true
  end

  test "perform failing to become healthy" do
    # Prevent expected failures from outputting to terminal
    Thread.report_on_exception = false

-    Object.any_instance.stubs(:sleep) # No sleeping when retrying
+    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying

    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
@@ -64,8 +63,6 @@ class CliHealthcheckTest < CliTestCase
      run_command("perform")
    end
    assert_match "container not ready (unhealthy)", exception.message
-  ensure
-    Thread.report_on_exception = true
  end

  private
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -66,7 +66,7 @@ class CliMainTest < CliTestCase
      .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }

    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*args| args[0..1] == [:mkdir, ".kamal/lock-app"] }
+      .with { |*arg| arg[0..1] == [:mkdir, ".kamal/lock-app"] }
      .raises(RuntimeError, "mkdir: cannot create directory ‘kamal_lock-app’: File exists")

    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_debug)
@@ -75,8 +75,6 @@ class CliMainTest < CliTestCase
    assert_raises(Kamal::Cli::LockError) do
      run_command("deploy")
    end
-  ensure
-    Thread.report_on_exception = true
  end

  test "deploy error when locking" do
@@ -92,8 +90,6 @@ class CliMainTest < CliTestCase
    assert_raises(SSHKit::Runner::ExecuteError) do
      run_command("deploy")
    end
-  ensure
-    Thread.report_on_exception = true
  end

  test "deploy errors during outside section leave remove lock" do
@@ -177,17 +173,13 @@ class CliMainTest < CliTestCase
      assert_match /docker container ls --all --filter name=\^app-web-nonsense\$ --quiet/, output
      assert_match /The app version 'nonsense' is not available as a container/, output
    end
-  ensure
-    Thread.report_on_exception = true
  end

  test "rollback good version" do
-    SecureRandom.stubs(:hex).with(16).returns("12345678901234567890123456789012")
-    SecureRandom.stubs(:hex).with(6).returns("123456789012")
-
+    Object.any_instance.stubs(:sleep)
    [ "web", "workers" ].each do |role|
      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :container, :ls, "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false)
+        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false)
        .returns("").at_least_once
      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet")
@@ -201,25 +193,20 @@ class CliMainTest < CliTestCase
    end

    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}'", "app-web-123")
-      .returns("172.17.0.3").at_least_once
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .returns("corddirectory").at_least_once # health check

    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{index .Args 1 }}'", :traefik)
-      .returns("[--providers.docker --providers.file.directory=/var/run/traefik-config --providers.file.watch --log.level=DEBUG --accesslog --accesslog.format=json]").at_least_once
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :exec, :traefik, :wget, "-qSO", "/dev/null", "http://localhost:80/up", "2>&1", "|", :grep, "-i", "X-Kamal-Run-ID", "|", :cut, "-d ' ' -f 4")
-      .returns("12345678901234567890123456789012").at_least_once
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-version-to-rollback$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("unhealthy").at_least_once # health check

    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
    hook_variables = { version: 123, service_version: "app@123", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "rollback" }

    run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
-      assert_match "Start container with version 123", output
      assert_hook_ran "pre-deploy", output, **hook_variables
      assert_match "docker tag dhh/app:123 dhh/app:latest", output
-      assert_match "docker start app-web-123", output
+      assert_match "docker run --detach --restart unless-stopped --name app-web-123", output
      assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running"
      assert_hook_ran "post-deploy", output, **hook_variables, runtime: "0"
    end
@@ -228,15 +215,10 @@ class CliMainTest < CliTestCase
  test "rollback without old version" do
    Kamal::Cli::Main.any_instance.stubs(:container_available?).returns(true)

-    Object.stubs(:sleep)
-    SecureRandom.stubs(:hex).with(16).returns("12345678901234567890123456789012")
-    SecureRandom.stubs(:hex).with(6).returns("123456789012")
+    Kamal::Utils::HealthcheckPoller.stubs(:sleep)

    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{index .Args 1 }}'", :traefik)
-      .returns("[--providers.docker --providers.file.directory=/var/run/traefik-config --providers.file.watch --log.level=DEBUG --accesslog --accesslog.format=json]").at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
      .returns("").at_least_once
    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
@@ -244,17 +226,9 @@ class CliMainTest < CliTestCase
    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
      .returns("running").at_least_once # health check
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}'", "app-web-123")
-      .returns("172.17.0.3").at_least_once
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :exec, :traefik, :wget, "-qSO", "/dev/null", "http://localhost:80/up", "2>&1", "|", :grep, "-i", "X-Kamal-Run-ID", "|", :cut, "-d ' ' -f 4")
-      .returns("12345678901234567890123456789012").at_least_once

    run_command("rollback", "123").tap do |output|
-      assert_match "Start container with version 123", output
-      assert_match "docker start app-web-123 || docker run --detach --restart unless-stopped --name app-web-123", output
+      assert_match "docker run --detach --restart unless-stopped --name app-web-123", output
      assert_no_match "docker stop", output
    end
  end
--- a/test/cli/traefik_test.rb
+++ b/test/cli/traefik_test.rb
@@ -4,7 +4,7 @@ class CliTraefikTest < CliTestCase
  test "boot" do
    run_command("boot").tap do |output|
      assert_match "docker login", output
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Configuration::Traefik::Static::DEFAULT_IMAGE} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\"", output
+      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
    end
  end

@@ -14,7 +14,7 @@ class CliTraefikTest < CliTestCase
    run_command("reboot").tap do |output|
      assert_match "docker container stop traefik", output
      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik", output
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Configuration::Traefik::Static::DEFAULT_IMAGE} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\"", output
+      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
    end
  end

--- a/test/commands/app_test.rb
+++ b/test/commands/app_test.rb
@@ -3,6 +3,7 @@ require "test_helper"
 class CommandsAppTest < ActiveSupport::TestCase
  setup do
    ENV["RAILS_MASTER_KEY"] = "456"
+    Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")

    @config = { service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], env: { "secret" => [ "RAILS_MASTER_KEY" ] } }
  end
@@ -13,13 +14,13 @@ class CommandsAppTest < ActiveSupport::TestCase

  test "run" do
    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

  test "run with hostname" do
    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run(hostname: "myhost").join(" ")
  end

@@ -27,7 +28,7 @@ class CommandsAppTest < ActiveSupport::TestCase
    @config[:volumes] = ["/local/path:/container/path" ]

    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

@@ -35,7 +36,7 @@ class CommandsAppTest < ActiveSupport::TestCase
    @config[:healthcheck] = { "path" => "/healthz" }

    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/healthz || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

@@ -43,7 +44,7 @@ class CommandsAppTest < ActiveSupport::TestCase
    @config[:healthcheck] = { "cmd" => "/bin/up" }

    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/up\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

@@ -51,7 +52,7 @@ class CommandsAppTest < ActiveSupport::TestCase
    @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } }

    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/healthy\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

@@ -66,7 +67,7 @@ class CommandsAppTest < ActiveSupport::TestCase
    @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }

    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

@@ -83,18 +84,6 @@ class CommandsAppTest < ActiveSupport::TestCase
      new_command.start.join(" ")
  end

-  test "start_or_run" do
-    assert_equal \
-      "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" dhh/app:999",
-      new_command.start_or_run.join(" ")
-  end
-
-  test "start_or_run with hostname" do
-    assert_equal \
-      "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" dhh/app:999",
-      new_command.start_or_run(hostname: "myhost").join(" ")
-  end
-
  test "stop" do
    assert_equal \
      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop",
@@ -342,6 +331,20 @@ class CommandsAppTest < ActiveSupport::TestCase
    assert_equal "rm -f .kamal/env/roles/app-web.env", new_command.remove_env_file.join(" ")
  end

+  test "cord" do
+    assert_equal "docker inspect -f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}' app-web-123 | awk '$2 == \"/tmp/kamal-cord\" {print $1}'", new_command.cord(version: 123).join(" ")
+  end
+
+  test "tie cord" do
+    assert_equal "mkdir -p . ; touch cordfile", new_command.tie_cord("cordfile").join(" ")
+    assert_equal "mkdir -p corddir ; touch corddir/cordfile", new_command.tie_cord("corddir/cordfile").join(" ")
+    assert_equal "mkdir -p /corddir ; touch /corddir/cordfile", new_command.tie_cord("/corddir/cordfile").join(" ")
+  end
+
+  test "cut cord" do
+    assert_equal "rm -r corddir", new_command.cut_cord("corddir").join(" ")
+  end
+
  private
    def new_command(role: "web")
      Kamal::Commands::App.new(Kamal::Configuration.new(@config, destination: @destination, version: "999"), role: role)
--- a/test/commands/healthcheck_test.rb
+++ b/test/commands/healthcheck_test.rb
@@ -40,8 +40,9 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase

  test "run with custom options" do
    @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere" } } }
+    @config[:healthcheck] = { "exposed_port" => 4999 }
    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 4999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
      new_command.run.join(" ")
  end

--- a/test/commands/traefik/static_test.rb
+++ b/test/commands/traefik/static_test.rb
@@ -1,6 +1,6 @@
 require "test_helper"

-class CommandsTraefikStaticTest < ActiveSupport::TestCase
+class CommandsTraefikTest < ActiveSupport::TestCase
  setup do
    @image = "traefik:test"

@@ -18,72 +18,72 @@ class CommandsTraefikStaticTest < ActiveSupport::TestCase

  test "run" do
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")

    @config[:traefik]["host_port"] = "8080"
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")

    @config[:traefik]["publish"] = false
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")
  end

  test "run with ports configured" do
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")

    @config[:traefik]["options"] = {"publish" => %w[9000:9000 9001:9001]}
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")
  end

  test "run with volumes configured" do
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")

    @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json] }
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")
  end

  test "run with several options configured" do
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")

    @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json], "publish" => %w[8080:8080], "memory" => "512m"}
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")
  end

  test "run with labels configured" do
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")

    @config[:traefik]["labels"] = { "traefik.http.routers.dashboard.service" => "api@internal", "traefik.http.routers.dashboard.middlewares" => "auth" }
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.dashboard.service=\"api@internal\" --label traefik.http.routers.dashboard.middlewares=\"auth\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.dashboard.service=\"api@internal\" --label traefik.http.routers.dashboard.middlewares=\"auth\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")
  end

  test "run with env configured" do
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")

    @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")
  end

@@ -91,7 +91,7 @@ class CommandsTraefikStaticTest < ActiveSupport::TestCase
    @config.delete(:traefik)

    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Configuration::Traefik::Static::DEFAULT_IMAGE} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"",
      new_command.run.join(" ")
  end

@@ -99,7 +99,7 @@ class CommandsTraefikStaticTest < ActiveSupport::TestCase
    @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }

    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")
  end

@@ -107,7 +107,7 @@ class CommandsTraefikStaticTest < ActiveSupport::TestCase
    @config[:traefik]["args"]["log.level"] = "ERROR"

    assert_equal \
-    "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/traefik-config:/var/run/traefik-config --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --providers.file.directory=\"/var/run/traefik-config\" --providers.file.watch --log.level=\"ERROR\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+    "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"ERROR\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")
  end

@@ -177,6 +177,16 @@ class CommandsTraefikStaticTest < ActiveSupport::TestCase
      new_command.follow_logs(host: @config[:servers].first, grep: 'hello!')
  end

+  test "env_file" do
+    @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
+
+    assert_equal "EXAMPLE_API_KEY=456\n", new_command.env_file
+  end
+
+  test "host_env_file_path" do
+    assert_equal ".kamal/env/traefik/traefik.env", new_command.host_env_file_path
+  end
+
  test "make_env_directory" do
    assert_equal "mkdir -p .kamal/env/traefik", new_command.make_env_directory.join(" ")
  end
@@ -187,6 +197,6 @@ class CommandsTraefikStaticTest < ActiveSupport::TestCase

  private
    def new_command
-      Kamal::Commands::Traefik::Static.new(Kamal::Configuration.new(@config, version: "123"))
+      Kamal::Commands::Traefik.new(Kamal::Configuration.new(@config, version: "123"))
    end
 end
--- a/test/configuration/role_test.rb
+++ b/test/configuration/role_test.rb
@@ -42,7 +42,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
  end

  test "special label args for web" do
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\""], @config.role(:web).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args
  end

  test "custom labels" do
@@ -66,7 +66,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
      c[:servers]["beta"] = { "traefik" => "true", "hosts" => [ "1.1.1.5" ] }
    })

-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"" ], config.role(:beta).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "traefik.http.services.app-beta.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args
  end

  test "env overwritten by role" do
@@ -175,4 +175,23 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
    assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).host_env_file_path
  end

+  test "uses cord" do
+    assert @config_with_roles.role(:web).uses_cord?
+    assert !@config_with_roles.role(:workers).uses_cord?
+  end
+
+  test "cord host file" do
+    assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}/cord}, @config_with_roles.role(:web).cord_host_file
+  end
+
+  test "cord volume" do
+    assert_equal "/tmp/kamal-cord", @config_with_roles.role(:web).cord_volume.container_path
+    assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}}, @config_with_roles.role(:web).cord_volume.host_path
+    assert_equal "--volume", @config_with_roles.role(:web).cord_volume.docker_args[0]
+    assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}:/tmp/kamal-cord}, @config_with_roles.role(:web).cord_volume.docker_args[1]
+  end
+
+  test "cord container file" do
+    assert_equal "/tmp/kamal-cord/cord", @config_with_roles.role(:web).cord_container_file
+  end
 end
--- a/test/configuration/traefik/static_test.rb
+++ b/test/configuration/traefik/static_test.rb
@@ -1,26 +0,0 @@
-require "test_helper"
-
-class ConfigurationTraefikStaticTest < ActiveSupport::TestCase
-  setup do
-    @deploy = {
-      service: "app", image: "dhh/app",
-      registry: { "username" => "dhh", "password" => "secret" },
-      env: { "REDIS_URL" => "redis://x/y" },
-      servers: [ "1.1.1.1", "1.1.1.2" ],
-      volumes: ["/local/path:/container/path"],
-      traefik: {}
-    }
-
-    @config = Kamal::Configuration.new(@deploy)
-  end
-
-  test "env_file" do
-    ENV["EXAMPLE_API_KEY"] = "456"
-    @config.traefik["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
-    traefik_static = Kamal::Configuration::Traefik::Static.new(config: @config)
-
-    assert_equal "EXAMPLE_API_KEY=456\n", traefik_static.env_file
-  ensure
-    ENV.delete "EXAMPLE_API_KEY"
-  end
-end
--- a/test/configuration_test.rb
+++ b/test/configuration_test.rb
@@ -224,7 +224,7 @@ class ConfigurationTest < ActiveSupport::TestCase
        :volume_args=>["--volume", "/local/path:/container/path"],
        :builder=>{},
        :logging=>["--log-opt", "max-size=\"10m\""],
-        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7 }}
+        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }}

    assert_equal expected_config, @config.to_h
  end
@@ -253,9 +253,16 @@ class ConfigurationTest < ActiveSupport::TestCase
    assert_equal "/root/kamal", config.run_directory
  end

-  test "app traefik labels raise ArgumentError" do
-    assert_raises(ArgumentError) do
-      Kamal::Configuration.new(@deploy.tap { |c| c.merge!(labels: { "traefik.enabled": true }) })
-    end
+  test "run directory as docker volume" do
+    config = Kamal::Configuration.new(@deploy)
+    assert_equal "$(pwd)/.kamal", config.run_directory_as_docker_volume
+
+    config = Kamal::Configuration.new(@deploy.merge!(run_directory: "/root/kamal"))
+    assert_equal "/root/kamal", config.run_directory_as_docker_volume
+  end
+
+  test "run id" do
+    SecureRandom.expects(:hex).with(16).returns("09876543211234567890098765432112")
+    assert_equal "09876543211234567890098765432112", @config.run_id
  end
 end
--- a/test/integration/docker/deployer/Dockerfile
+++ b/test/integration/docker/deployer/Dockerfile
@@ -2,6 +2,8 @@ FROM ruby:3.2

 WORKDIR /app

+ENV VERBOSE=true
+
 RUN apt-get update --fix-missing && apt-get install -y ca-certificates openssh-client curl gnupg docker.io

 RUN install -m 0755 -d /etc/apt/keyrings
--- a/test/integration/docker/deployer/app/Dockerfile
+++ b/test/integration/docker/deployer/app/Dockerfile
@@ -4,4 +4,5 @@ COPY default.conf /etc/nginx/conf.d/default.conf

 ARG COMMIT_SHA
 RUN echo $COMMIT_SHA > /usr/share/nginx/html/version
+RUN mkdir -p /usr/share/nginx/html/versions && echo "version" > /usr/share/nginx/html/versions/$COMMIT_SHA

--- a/test/integration/docker/deployer/app/config/deploy.yml
+++ b/test/integration/docker/deployer/app/config/deploy.yml
@@ -8,6 +8,7 @@ env:
    CLEAR_TOKEN: '4321'
  secret:
    - SECRET_TOKEN
+asset_path: /usr/share/nginx/html/versions

 registry:
  server: registry:4443
@@ -18,8 +19,7 @@ builder:
  args:
    COMMIT_SHA: <%= `git rev-parse HEAD` %>
 healthcheck:
-  cmd: wget -qO- http://localhost > /dev/null
-  path: /
+  cmd: wget -qO- http://localhost > /dev/null || exit 1
 traefik:
  args:
    accesslog: true
--- a/test/integration/main_test.rb
+++ b/test/integration/main_test.rb
@@ -20,6 +20,8 @@ class MainTest < IntegrationTest
    assert_app_is_up version: second_version
    assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy"

+    assert_accumulated_assets first_version, second_version
+
    kamal :rollback, first_version
    assert_hooks_ran "pre-connect", "pre-deploy", "post-deploy"
    assert_app_is_up version: first_version
@@ -39,12 +41,6 @@ class MainTest < IntegrationTest
    assert_no_remote_env_file
  end

-  test "envify" do
-    kamal :envify
-
-    assert_equal "SECRET_TOKEN=1234", deployer_exec("cat .env", capture: true)
-  end
-
  test "config" do
    config = YAML.load(kamal(:config, capture: true))
    version = latest_app_version
@@ -60,7 +56,7 @@ class MainTest < IntegrationTest
    assert_equal({ user: "root", auth_methods: [ "publickey" ], keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])
    assert_equal({ "multiarch" => false, "args" => { "COMMIT_SHA" => version } }, config[:builder])
    assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging]
-    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "cmd" => "wget -qO- http://localhost > /dev/null" }, config[:healthcheck])
+    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord"=>"/tmp/kamal-cord", "cmd"=>"wget -qO- http://localhost > /dev/null || exit 1" }, config[:healthcheck])
  end

  private
@@ -75,4 +71,10 @@ class MainTest < IntegrationTest
    def assert_no_remote_env_file
      assert_equal "nofile", docker_compose("exec vm1 stat /root/.kamal/env/roles/app-web.env 2> /dev/null || echo nofile", capture: true)
    end
+
+    def assert_accumulated_assets(*versions)
+      versions.each do |version|
+        assert_equal "200", Net::HTTP.get_response(URI.parse("http://localhost:12345/versions/#{version}")).code
+      end
+    end
 end
Author	SHA1	Message	Date
Donal McBreen	c122f97181	WIP	2023-09-08 16:40:41 +01:00
Donal McBreen	aa9999809c	Merge pull request #439 from basecamp/zero-downtime-deploy-file Zero downtime deployment with cord file	2023-09-07 09:34:40 +01:00
Donal McBreen	6263bf96ba	Merge pull request #438 from basecamp/remote-env-file Copy env files to remote hosts	2023-09-07 09:34:22 +01:00
Donal McBreen	8a41d15b69	Zero downtime deployment with cord file When replacing a container currently we: 1. Boot the new container 2. Wait for it to become healthy 3. Stop the old container Traefik will send requests to the old container until it notices that it is unhealthy. But it may have stopped serving requests before that point which can result in errors. To get round that the new boot process is: 1. Create a directory with a single file on the host 2. Boot the new container, mounting the cord file into /tmp and including a check for the file in the docker healthcheck 3. Wait for it to become healthy 4. Delete the healthcheck file ("cut the cord") for the old container 5. Wait for it to become unhealthy and give Traefik a couple of seconds to notice 6. Stop the old container The extra steps ensure that Traefik stops sending requests before the old container is shutdown.	2023-09-06 14:35:30 +01:00
Donal McBreen	94bf090657	Copy env files to remote hosts Setting env variables in the docker arguments requires having them on the deploy host. Instead we'll add two new commands `kamal env push` and `kamal env delete` which will manage copying the environment as .env files to the remote host. Docker will pick up the file with `--env-file <path-to-file>`. Env files will be stored under `<kamal run directory>/env`. Running `kamal env push` will create env files for each role and accessory, and traefik if required. `kamal envify` has been updated to also push the env files. By avoiding using `kamal envify` and creating the local and remote secrets manually, you can now avoid accessing secrets needed for the docker runtime environment locally. You will still need build secrets. One thing to note - the Docker doesn't parse the environment variables in the env file, one result of this is that you can't specify multi-line values - see https://github.com/moby/moby/issues/12997. We maybe need to look docker config or docker secrets longer term to get around this. Hattip to @kevinmcconnell - this was all his idea.	2023-09-06 14:33:13 +01:00
Donal McBreen	adc7173cf2	Merge pull request #437 from basecamp/kamal-run-directory Configurable Kamal directory	2023-09-06 14:31:07 +01:00
Donal McBreen	fd6bf5324a	Merge pull request #443 from rience/custom-healthcheck-port Configurable Healthcheck Expose Port	2023-09-06 11:09:48 +01:00
Krzysztof Adamski	c2b2f7ea33	Fixing Tests	2023-09-06 10:16:59 +02:00
Krzysztof Adamski	bbcc90e4d1	Configurable Healthcheck Expose Port	2023-09-05 10:53:32 +02:00