Zero downtime deployment with cord file

When replacing a container currently we: 1. Boot the new container 2. Wait for it to become healthy 3. Stop the old container Traefik will send requests to the old container until it notices that it is unhealthy. But it may have stopped serving requests before that point which can result in errors. To get round that the new boot process is: 1. Create a directory with a single file on the host 2. Boot the new container, mounting the cord file into /tmp and including a check for the file in the docker healthcheck 3. Wait for it to become healthy 4. Delete the healthcheck file ("cut the cord") for the old container 5. Wait for it to become unhealthy and give Traefik a couple of seconds to notice 6. Stop the old container The extra steps ensure that Traefik stops sending requests before the old container is shutdown.
2023-08-31 10:21:57 +01:00
parent 94bf090657
commit 8a41d15b69
17 changed files with 234 additions and 65 deletions
--- a/lib/kamal/cli/app.rb
+++ b/lib/kamal/cli/app.rb
@@ -18,8 +18,9 @@ class Kamal::Cli::App < Kamal::Cli::Base
            roles.each do |role|
              app = KAMAL.app(role: role)
              auditor = KAMAL.auditor(role: role)
+              role_config = KAMAL.config.role(role)

-              if capture_with_info(*app.container_id_for_version(version, only_running: true), raise_on_non_zero_exit: false).present?
+              if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
                tmp_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
                info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
                execute *auditor.record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
@@ -29,11 +30,25 @@ class Kamal::Cli::App < Kamal::Cli::Base
              execute *auditor.record("Booted app version #{version}"), verbosity: :debug

              old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-              execute *app.start_or_run(hostname: "#{host}-#{SecureRandom.hex(6)}")
+
+              if role_config.uses_cord?
+                execute *app.tie_cord(role_config.cord_host_file)
+              end
+
+              execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")

              Kamal::Utils::HealthcheckPoller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }

-              execute *app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
+              if old_version.present?
+                if role_config.uses_cord?
+                  cord = capture_with_info(*app.cord(version: old_version), raise_on_non_zero_exit: false).strip
+                  if cord.present?
+                    execute *app.cut_cord(cord)
+                    Kamal::Utils::HealthcheckPoller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
+                  end
+                end
+                execute *app.stop(version: old_version), raise_on_non_zero_exit: false
+              end
            end
          end
        end
--- a/lib/kamal/commands/app.rb
+++ b/lib/kamal/commands/app.rb
@@ -1,34 +1,29 @@
 class Kamal::Commands::App < Kamal::Commands::Base
  ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]

-  attr_reader :role
+  attr_reader :role, :role_config

  def initialize(config, role: nil)
    super(config)
    @role = role
-  end
-
-  def start_or_run(hostname: nil)
-    combine start, run(hostname: hostname), by: "||"
+    @role_config = config.role(self.role)
  end

  def run(hostname: nil)
-    role = config.role(self.role)
-
    docker :run,
      "--detach",
      "--restart unless-stopped",
      "--name", container_name,
      *(["--hostname", hostname] if hostname),
      "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
-      *role.env_args,
-      *role.health_check_args,
+      *role_config.env_args,
+      *role_config.health_check_args,
      *config.logging_args,
      *config.volume_args,
-      *role.label_args,
-      *role.option_args,
+      *role_config.label_args,
+      *role_config.option_args,
      config.absolute_image,
-      role.cmd
+      role_config.cmd
  end

  def start
@@ -76,14 +71,12 @@ class Kamal::Commands::App < Kamal::Commands::Base
  end

  def execute_in_new_container(*command, interactive: false)
-    role = config.role(self.role)
-
    docker :run,
      ("-it" if interactive),
      "--rm",
-      *role&.env_args,
+      *role_config&.env_args,
      *config.volume_args,
-      *role&.option_args,
+      *role_config&.option_args,
      config.absolute_image,
      *command
  end
@@ -112,7 +105,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
  def list_versions(*docker_args, statuses: nil)
    pipe \
      docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
-      %(while read line; do echo ${line##{service_role_dest}-}; done) # Extract SHA from "service-role-dest-SHA"
+      %(while read line; do echo ${line##{role_config.full_name}-}; done) # Extract SHA from "service-role-dest-SHA"
  end

  def list_containers
@@ -150,16 +143,30 @@ class Kamal::Commands::App < Kamal::Commands::Base
  end

  def make_env_directory
-    make_directory config.role(role).host_env_directory
+    make_directory role_config.host_env_directory
  end

  def remove_env_file
-    [:rm, "-f", config.role(role).host_env_file_path]
+    [:rm, "-f", role_config.host_env_file_path]
+  end
+
+  def cord(version:)
+    pipe \
+      docker(:inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", container_name(version)),
+      [:awk, "'$2 == \"#{role_config.cord_container_directory}\" {print $1}'"]
+  end
+
+  def tie_cord(cord)
+    create_empty_file(cord)
+  end
+
+  def cut_cord(cord)
+    remove_directory(cord)
  end

  private
    def container_name(version = nil)
-      [ config.service, role, config.destination, version || config.version ].compact.join("-")
+      [ role_config.full_name, version || config.version ].compact.join("-")
    end

    def filter_args(statuses: nil)
--- a/lib/kamal/commands/base.rb
+++ b/lib/kamal/commands/base.rb
@@ -34,6 +34,10 @@ module Kamal::Commands
      [ :mkdir, "-p", path ]
    end

+    def remove_directory(path)
+      [ :rm, "-r", path ]
+    end
+
    private
      def combine(*commands, by: "&&")
        commands
@@ -69,5 +73,11 @@ module Kamal::Commands
      def tags(**details)
        Kamal::Tags.from_config(config, **details)
      end
+
+      def create_empty_file(file)
+        chain \
+          make_directory_for(file),
+          [:touch, file]
+      end
  end
 end
--- a/lib/kamal/commands/healthcheck.rb
+++ b/lib/kamal/commands/healthcheck.rb
@@ -10,7 +10,7 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
      "--label", "service=#{container_name}",
      "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
      *web.env_args,
-      *web.health_check_args,
+      *web.health_check_args(cord: false),
      *config.volume_args,
      *web.option_args,
      config.absolute_image,
--- a/lib/kamal/configuration.rb
+++ b/lib/kamal/configuration.rb
@@ -61,6 +61,14 @@ class Kamal::Configuration
    raw_config.run_directory || ".kamal"
  end

+  def run_directory_as_docker_volume
+    if Pathname.new(run_directory).absolute?
+      run_directory
+    else
+      File.join "$(pwd)", run_directory
+    end
+  end
+

  def roles
    @roles ||= role_names.collect { |role_name| Role.new(role_name, config: self) }
@@ -141,7 +149,7 @@ class Kamal::Configuration


  def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999 }.merge(raw_config.healthcheck || {})
+    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }.merge(raw_config.healthcheck || {})
  end

  def readiness_delay
@@ -199,6 +207,10 @@ class Kamal::Configuration
    "#{run_directory}/env"
  end

+  def run_id
+    @run_id ||= SecureRandom.hex(16)
+  end
+
  private
    # Will raise ArgumentError if any required config keys are missing
    def ensure_required_keys_present
--- a/lib/kamal/configuration/role.rb
+++ b/lib/kamal/configuration/role.rb
@@ -1,4 +1,5 @@
 class Kamal::Configuration::Role
+  CORD_FILE = "cord"
  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils

  attr_accessor :name
@@ -47,28 +48,52 @@ class Kamal::Configuration::Role
    argumentize "--env-file", host_env_file_path
  end

-  def health_check_args
+  def health_check_args(cord: true)
    if health_check_cmd.present?
+      if cord && uses_cord?
+        optionize({ "health-cmd" => health_check_cmd_with_cord, "health-interval" => health_check_interval })
+          .concat(["--volume", "#{cord_host_directory}:#{cord_container_directory}"])
+      else
        optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval })
+      end
    else
      []
    end
  end

  def health_check_cmd
-    options = specializations["healthcheck"] || {}
-    options = config.healthcheck.merge(options) if running_traefik?
+    health_check_options["cmd"] || http_health_check(port: health_check_options["port"], path: health_check_options["path"])
+  end

-    options["cmd"] || http_health_check(port: options["port"], path: options["path"])
+  def health_check_cmd_with_cord
+    "(#{health_check_cmd}) && (stat #{cord_container_file} > /dev/null || exit 1)"
  end

  def health_check_interval
-    options = specializations["healthcheck"] || {}
-    options = config.healthcheck.merge(options) if running_traefik?
-
-    options["interval"] || "1s"
+    health_check_options["interval"] || "1s"
  end

+  def uses_cord?
+    running_traefik? && cord_container_directory.present? && health_check_cmd.present?
+  end
+
+  def cord_host_directory
+    File.join config.run_directory_as_docker_volume, "cords", [full_name, config.run_id].join("-")
+  end
+
+  def cord_host_file
+    File.join cord_host_directory, CORD_FILE
+  end
+
+  def cord_container_directory
+    health_check_options.fetch("cord", nil)
+  end
+
+  def cord_container_file
+    File.join cord_container_directory, CORD_FILE
+  end
+
+
  def cmd
    specializations["cmd"]
  end
@@ -85,6 +110,10 @@ class Kamal::Configuration::Role
    name.web? || specializations["traefik"]
  end

+  def full_name
+    [ config.service, name, config.destination ].compact.join("-")
+  end
+
  private
    attr_accessor :config

@@ -164,4 +193,12 @@ class Kamal::Configuration::Role
    def http_health_check(port:, path:)
      "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present?
    end
+
+    def health_check_options
+      @health_check_options ||= begin
+        options = specializations["healthcheck"] || {}
+        options = config.healthcheck.merge(options) if running_traefik?
+        options
+      end
+    end
 end
--- a/lib/kamal/utils/healthcheck_poller.rb
+++ b/lib/kamal/utils/healthcheck_poller.rb
@@ -1,5 +1,5 @@
 class Kamal::Utils::HealthcheckPoller
-  TRAEFIK_HEALTHY_DELAY = 2
+  TRAEFIK_UPDATE_DELAY = 2

  class HealthcheckError < StandardError; end

@@ -11,7 +11,7 @@ class Kamal::Utils::HealthcheckPoller
      begin
        case status = block.call
        when "healthy"
-          sleep TRAEFIK_HEALTHY_DELAY if pause_after_ready
+          sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
        when "running" # No health check configured
          sleep KAMAL.config.readiness_delay if pause_after_ready
        else
@@ -31,6 +31,31 @@ class Kamal::Utils::HealthcheckPoller
      info "Container is healthy!"
    end

+    def wait_for_unhealthy(pause_after_ready: false, &block)
+      attempt = 1
+      max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+      begin
+        case status = block.call
+        when "unhealthy"
+          sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
+        else
+          raise HealthcheckError, "container not unhealthy (#{status})"
+        end
+      rescue HealthcheckError => e
+        if attempt <= max_attempts
+          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+          sleep attempt
+          attempt += 1
+          retry
+        else
+          raise
+        end
+      end
+
+      info "Container is unhealthy!"
+    end
+
    private
      def info(message)
        SSHKit.config.output.info(message)
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -11,10 +11,11 @@ class CliAppTest < CliTestCase
  end

  test "boot will rename if same version is already running" do
+    Object.any_instance.stubs(:sleep)
    run_command("details") # Preheat Kamal const

    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
      .returns("12345678") # running version

    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -25,6 +26,14 @@ class CliAppTest < CliTestCase
      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
      .returns("123") # old version

+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .returns("cordfile") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("unhealthy") # old version unhealthy
+
    run_command("boot").tap do |output|
      assert_match /Renaming container .* to .* as already deployed on 1.1.1.1/, output # Rename
      assert_match /docker rename app-web-latest app-web-latest_replaced_[0-9a-f]{16}/, output
@@ -180,10 +189,16 @@ class CliAppTest < CliTestCase
    end

    def stub_running
+      Object.any_instance.stubs(:sleep)
+
      SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version

      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
        .returns("running") # health check
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+        .returns("unhealthy") # health check
    end
 end
--- a/test/cli/healthcheck_test.rb
+++ b/test/cli/healthcheck_test.rb
@@ -6,6 +6,7 @@ class CliHealthcheckTest < CliTestCase
    Thread.report_on_exception = false

    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")

    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -176,9 +176,10 @@ class CliMainTest < CliTestCase
  end

  test "rollback good version" do
+    Object.any_instance.stubs(:sleep)
    [ "web", "workers" ].each do |role|
      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :container, :ls, "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false)
+        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false)
        .returns("").at_least_once
      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet")
@@ -191,14 +192,21 @@ class CliMainTest < CliTestCase
        .returns("running").at_least_once # health check
    end

+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .returns("corddirectory").at_least_once # health check
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-version-to-rollback$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("unhealthy").at_least_once # health check
+
    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
    hook_variables = { version: 123, service_version: "app@123", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "rollback" }

    run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
-      assert_match "Start container with version 123", output
      assert_hook_ran "pre-deploy", output, **hook_variables
      assert_match "docker tag dhh/app:123 dhh/app:latest", output
-      assert_match "docker start app-web-123", output
+      assert_match "docker run --detach --restart unless-stopped --name app-web-123", output
      assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running"
      assert_hook_ran "post-deploy", output, **hook_variables, runtime: "0"
    end
@@ -210,7 +218,7 @@ class CliMainTest < CliTestCase
    Kamal::Utils::HealthcheckPoller.stubs(:sleep)

    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
      .returns("").at_least_once
    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
@@ -220,8 +228,7 @@ class CliMainTest < CliTestCase
      .returns("running").at_least_once # health check

    run_command("rollback", "123").tap do |output|
-      assert_match "Start container with version 123", output
-      assert_match "docker start app-web-123 || docker run --detach --restart unless-stopped --name app-web-123", output
+      assert_match "docker run --detach --restart unless-stopped --name app-web-123", output
      assert_no_match "docker stop", output
    end
  end
--- a/test/cli/traefik_test.rb
+++ b/test/cli/traefik_test.rb
@@ -19,6 +19,8 @@ class CliTraefikTest < CliTestCase
  end

  test "reboot --rolling" do
+    Object.any_instance.stubs(:sleep)
+
    run_command("reboot", "--rolling").tap do |output|
      assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.1", output
    end
--- a/test/commands/app_test.rb
+++ b/test/commands/app_test.rb
@@ -3,6 +3,7 @@ require "test_helper"
 class CommandsAppTest < ActiveSupport::TestCase
  setup do
    ENV["RAILS_MASTER_KEY"] = "456"
+    Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")

    @config = { service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], env: { "secret" => [ "RAILS_MASTER_KEY" ] } }
  end
@@ -13,13 +14,13 @@ class CommandsAppTest < ActiveSupport::TestCase

  test "run" do
    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

  test "run with hostname" do
    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run(hostname: "myhost").join(" ")
  end

@@ -27,7 +28,7 @@ class CommandsAppTest < ActiveSupport::TestCase
    @config[:volumes] = ["/local/path:/container/path" ]

    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

@@ -35,7 +36,7 @@ class CommandsAppTest < ActiveSupport::TestCase
    @config[:healthcheck] = { "path" => "/healthz" }

    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/healthz || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

@@ -43,7 +44,7 @@ class CommandsAppTest < ActiveSupport::TestCase
    @config[:healthcheck] = { "cmd" => "/bin/up" }

    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/up\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

@@ -51,7 +52,7 @@ class CommandsAppTest < ActiveSupport::TestCase
    @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } }

    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/healthy\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

@@ -66,7 +67,7 @@ class CommandsAppTest < ActiveSupport::TestCase
    @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }

    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

@@ -83,18 +84,6 @@ class CommandsAppTest < ActiveSupport::TestCase
      new_command.start.join(" ")
  end

-  test "start_or_run" do
-    assert_equal \
-      "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
-      new_command.start_or_run.join(" ")
-  end
-
-  test "start_or_run with hostname" do
-    assert_equal \
-      "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
-      new_command.start_or_run(hostname: "myhost").join(" ")
-  end
-
  test "stop" do
    assert_equal \
      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop",
@@ -342,6 +331,20 @@ class CommandsAppTest < ActiveSupport::TestCase
    assert_equal "rm -f .kamal/env/roles/app-web.env", new_command.remove_env_file.join(" ")
  end

+  test "cord" do
+    assert_equal "docker inspect -f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}' app-web-123 | awk '$2 == \"/tmp/kamal-cord\" {print $1}'", new_command.cord(version: 123).join(" ")
+  end
+
+  test "tie cord" do
+    assert_equal "mkdir -p . ; touch cordfile", new_command.tie_cord("cordfile").join(" ")
+    assert_equal "mkdir -p corddir ; touch corddir/cordfile", new_command.tie_cord("corddir/cordfile").join(" ")
+    assert_equal "mkdir -p /corddir ; touch /corddir/cordfile", new_command.tie_cord("/corddir/cordfile").join(" ")
+  end
+
+  test "cut cord" do
+    assert_equal "rm -r corddir", new_command.cut_cord("corddir").join(" ")
+  end
+
  private
    def new_command(role: "web")
      Kamal::Commands::App.new(Kamal::Configuration.new(@config, destination: @destination, version: "999"), role: role)
--- a/test/configuration/role_test.rb
+++ b/test/configuration/role_test.rb
@@ -175,4 +175,24 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
    assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).host_env_file_path
  end

+  test "uses cord" do
+    assert @config_with_roles.role(:web).uses_cord?
+    assert !@config_with_roles.role(:workers).uses_cord?
+  end
+
+  test "cord host directory" do
+    assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}}, @config_with_roles.role(:web).cord_host_directory
+  end
+
+  test "cord host file" do
+    assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}/cord}, @config_with_roles.role(:web).cord_host_file
+  end
+
+  test "cord container directory" do
+    assert_equal "/tmp/kamal-cord", @config_with_roles.role(:web).cord_container_directory
+  end
+
+  test "cord container file" do
+    assert_equal "/tmp/kamal-cord/cord", @config_with_roles.role(:web).cord_container_file
+  end
 end
--- a/test/configuration_test.rb
+++ b/test/configuration_test.rb
@@ -224,7 +224,7 @@ class ConfigurationTest < ActiveSupport::TestCase
        :volume_args=>["--volume", "/local/path:/container/path"],
        :builder=>{},
        :logging=>["--log-opt", "max-size=\"10m\""],
-        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999 }}
+        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }}

    assert_equal expected_config, @config.to_h
  end
@@ -252,4 +252,17 @@ class ConfigurationTest < ActiveSupport::TestCase
    config = Kamal::Configuration.new(@deploy.merge!(run_directory: "/root/kamal"))
    assert_equal "/root/kamal", config.run_directory
  end
+
+  test "run directory as docker volume" do
+    config = Kamal::Configuration.new(@deploy)
+    assert_equal "$(pwd)/.kamal", config.run_directory_as_docker_volume
+
+    config = Kamal::Configuration.new(@deploy.merge!(run_directory: "/root/kamal"))
+    assert_equal "/root/kamal", config.run_directory_as_docker_volume
+  end
+
+  test "run id" do
+    SecureRandom.expects(:hex).with(16).returns("09876543211234567890098765432112")
+    assert_equal "09876543211234567890098765432112", @config.run_id
+  end
 end
--- a/test/integration/docker/deployer/Dockerfile
+++ b/test/integration/docker/deployer/Dockerfile
@@ -2,6 +2,8 @@ FROM ruby:3.2

 WORKDIR /app

+ENV VERBOSE=true
+
 RUN apt-get update --fix-missing && apt-get install -y ca-certificates openssh-client curl gnupg docker.io

 RUN install -m 0755 -d /etc/apt/keyrings
--- a/test/integration/docker/deployer/app/config/deploy.yml
+++ b/test/integration/docker/deployer/app/config/deploy.yml
@@ -18,7 +18,7 @@ builder:
  args:
    COMMIT_SHA: <%= `git rev-parse HEAD` %>
 healthcheck:
-  cmd: wget -qO- http://localhost > /dev/null
+  cmd: wget -qO- http://localhost > /dev/null || exit 1
 traefik:
  args:
    accesslog: true
--- a/test/integration/main_test.rb
+++ b/test/integration/main_test.rb
@@ -54,7 +54,7 @@ class MainTest < IntegrationTest
    assert_equal({ user: "root", auth_methods: [ "publickey" ], keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])
    assert_equal({ "multiarch" => false, "args" => { "COMMIT_SHA" => version } }, config[:builder])
    assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging]
-    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cmd" => "wget -qO- http://localhost > /dev/null" }, config[:healthcheck])
+    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord"=>"/tmp/kamal-cord", "cmd"=>"wget -qO- http://localhost > /dev/null || exit 1" }, config[:healthcheck])
  end

  private