From bcfa1d83e8aac0fb8bb8b2c39a5558d591c4c972 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Mon, 28 Aug 2023 16:12:56 +0100
Subject: [PATCH 1/6] Configurable Kamal directory

To avoid polluting the default SSH directory with lots of Kamal config,
we'll default to putting them in a `kamal` sub directory.

But also make the directory configurable with the `run_directory` key,
so for example you can set it as `/var/run/kamal/`

The directory is created during bootstrap or before any command that
will need to access a file.
---
 lib/kamal/cli/base.rb         |  8 ++++++++
 lib/kamal/cli/lock.rb         | 15 ++++++++++++---
 lib/kamal/cli/server.rb       |  4 ++++
 lib/kamal/commander.rb        |  4 ++++
 lib/kamal/commands/auditor.rb |  4 +++-
 lib/kamal/commands/lock.rb    |  2 +-
 lib/kamal/commands/server.rb  |  5 +++++
 lib/kamal/configuration.rb    |  4 ++++
 test/cli/build_test.rb        |  6 +++---
 test/cli/cli_test_case.rb     |  8 +++++---
 test/cli/main_test.rb         | 14 ++++++++++----
 test/cli/server_test.rb       |  3 +++
 test/cli/traefik_test.rb      |  2 +-
 test/commands/auditor_test.rb |  8 ++++----
 test/commands/lock_test.rb    |  6 +++---
 test/commands/server_test.rb  | 23 +++++++++++++++++++++++
 test/configuration_test.rb    |  8 ++++++++
 17 files changed, 101 insertions(+), 23 deletions(-)
 create mode 100644 lib/kamal/commands/server.rb
 create mode 100644 test/commands/server_test.rb

diff --git a/lib/kamal/cli/base.rb b/lib/kamal/cli/base.rb
index 9f38241a..8bbdf198 100644
--- a/lib/kamal/cli/base.rb
+++ b/lib/kamal/cli/base.rb
@@ -79,6 +79,8 @@ module Kamal::Cli
 
         run_hook "pre-connect"
 
+        ensure_run_directory
+
         acquire_lock
 
         begin
@@ -167,5 +169,11 @@ module Kamal::Cli
       def first_invocation
         instance_variable_get("@_invocations").first
       end
+
+      def ensure_run_directory
+        on(KAMAL.hosts) do
+          execute(*KAMAL.server.ensure_run_directory)
+        end
+      end
     end
 end
diff --git a/lib/kamal/cli/lock.rb b/lib/kamal/cli/lock.rb
index c30e6f0f..1e4b52cf 100644
--- a/lib/kamal/cli/lock.rb
+++ b/lib/kamal/cli/lock.rb
@@ -2,7 +2,10 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   desc "status", "Report lock status"
   def status
     handle_missing_lock do
-      on(KAMAL.primary_host) { puts capture_with_debug(*KAMAL.lock.status) }
+      on(KAMAL.primary_host) do
+        execute *KAMAL.server.ensure_run_directory
+        puts capture_with_debug(*KAMAL.lock.status)
+      end
     end
   end
 
@@ -11,7 +14,10 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   def acquire
     message = options[:message]
     raise_if_locked do
-      on(KAMAL.primary_host) { execute *KAMAL.lock.acquire(message, KAMAL.config.version), verbosity: :debug }
+      on(KAMAL.primary_host) do
+        execute *KAMAL.server.ensure_run_directory
+        execute *KAMAL.lock.acquire(message, KAMAL.config.version), verbosity: :debug
+      end
       say "Acquired the deploy lock"
     end
   end
@@ -19,7 +25,10 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   desc "release", "Release the deploy lock"
   def release
     handle_missing_lock do
-      on(KAMAL.primary_host) { execute *KAMAL.lock.release, verbosity: :debug }
+      on(KAMAL.primary_host) do
+        execute *KAMAL.server.ensure_run_directory
+        execute *KAMAL.lock.release, verbosity: :debug
+      end
       say "Released the deploy lock"
     end
   end
diff --git a/lib/kamal/cli/server.rb b/lib/kamal/cli/server.rb
index 4cf23a5d..8387d47d 100644
--- a/lib/kamal/cli/server.rb
+++ b/lib/kamal/cli/server.rb
@@ -14,6 +14,10 @@ class Kamal::Cli::Server < Kamal::Cli::Base
       end
     end
 
+    on(KAMAL.hosts) do
+      execute(*KAMAL.server.ensure_run_directory)
+    end
+
     if missing.any?
       raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/"
     end
diff --git a/lib/kamal/commander.rb b/lib/kamal/commander.rb
index 04795164..47e0cf9f 100644
--- a/lib/kamal/commander.rb
+++ b/lib/kamal/commander.rb
@@ -116,6 +116,10 @@ class Kamal::Commander
     @registry ||= Kamal::Commands::Registry.new(config)
   end
 
+  def server
+    @server ||= Kamal::Commands::Server.new(config)
+  end
+
   def traefik
     @traefik ||= Kamal::Commands::Traefik.new(config)
   end
diff --git a/lib/kamal/commands/auditor.rb b/lib/kamal/commands/auditor.rb
index 0fefbbcb..ad9c4d16 100644
--- a/lib/kamal/commands/auditor.rb
+++ b/lib/kamal/commands/auditor.rb
@@ -19,7 +19,9 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
 
   private
     def audit_log_file
-      [ "kamal", config.service, config.destination, "audit.log" ].compact.join("-")
+      file = [ config.service, config.destination, "audit.log" ].compact.join("-")
+
+      "#{config.run_directory}/#{file}"
     end
 
     def audit_tags(**details)
diff --git a/lib/kamal/commands/lock.rb b/lib/kamal/commands/lock.rb
index 03b9abd2..c5d216ac 100644
--- a/lib/kamal/commands/lock.rb
+++ b/lib/kamal/commands/lock.rb
@@ -40,7 +40,7 @@ class Kamal::Commands::Lock < Kamal::Commands::Base
     end
 
     def lock_dir
-      "kamal_lock-#{config.service}"
+      "#{config.run_directory}/lock-#{config.service}"
     end
 
     def lock_details_file
diff --git a/lib/kamal/commands/server.rb b/lib/kamal/commands/server.rb
new file mode 100644
index 00000000..5b3ad194
--- /dev/null
+++ b/lib/kamal/commands/server.rb
@@ -0,0 +1,5 @@
+class Kamal::Commands::Server < Kamal::Commands::Base
+  def ensure_run_directory
+    [:mkdir, "-p", config.run_directory]
+  end
+end
diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb
index d6e8d757..af9a0eac 100644
--- a/lib/kamal/configuration.rb
+++ b/lib/kamal/configuration.rb
@@ -57,6 +57,10 @@ class Kamal::Configuration
     Kamal::Utils.abbreviate_version(version)
   end
 
+  def run_directory
+    raw_config.run_directory || "kamal"
+  end
+
 
   def roles
     @roles ||= role_names.collect { |role_name| Role.new(role_name, config: self) }
diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb
index 4e09564c..35e8b761 100644
--- a/test/cli/build_test.rb
+++ b/test/cli/build_test.rb
@@ -20,7 +20,7 @@ class CliBuildTest < CliTestCase
   end
 
   test "push without builder" do
-    stub_locking
+    stub_setup
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, "--version", "&&", :docker, :buildx, "version")
 
@@ -36,7 +36,7 @@ class CliBuildTest < CliTestCase
   end
 
   test "push with no buildx plugin" do
-    stub_locking
+    stub_setup
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, "--version", "&&", :docker, :buildx, "version")
       .raises(SSHKit::Command::Failed.new("no buildx"))
@@ -67,7 +67,7 @@ class CliBuildTest < CliTestCase
   end
 
   test "create with error" do
-    stub_locking
+    stub_setup
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with { |arg| arg == :docker }
       .raises(SSHKit::Command::Failed.new("stderr=error"))
diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb
index e28f9632..5aa1cc1b 100644
--- a/test/cli/cli_test_case.rb
+++ b/test/cli/cli_test_case.rb
@@ -27,11 +27,13 @@ class CliTestCase < ActiveSupport::TestCase
         .raises(SSHKit::Command::Failed.new("failed"))
     end
 
-    def stub_locking
+    def stub_setup
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :mkdir && arg2 == "kamal_lock-app" }
+        .with { |*args| args == [ :mkdir, "-p", "kamal" ] }
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :rm && arg2 == "kamal_lock-app/details" }
+        .with { |arg1, arg2| arg1 == :mkdir && arg2 == "kamal/lock-app" }
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |arg1, arg2| arg1 == :rm && arg2 == "kamal/lock-app/details" }
     end
 
     def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: nil)
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index 888a6688..d7afce4f 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -63,11 +63,14 @@ class CliMainTest < CliTestCase
     Thread.report_on_exception = false
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*arg| arg[0..1] == [:mkdir, 'kamal_lock-app'] }
+      .with { |*args| args == [ :mkdir, "-p", "kamal" ] }
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |*arg| arg[0..1] == [:mkdir, 'kamal/lock-app'] }
       .raises(RuntimeError, "mkdir: cannot create directory ‘kamal_lock-app’: File exists")
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_debug)
-      .with(:stat, 'kamal_lock-app', ">", "/dev/null", "&&", :cat, "kamal_lock-app/details", "|", :base64, "-d")
+      .with(:stat, 'kamal/lock-app', ">", "/dev/null", "&&", :cat, "kamal/lock-app/details", "|", :base64, "-d")
 
     assert_raises(Kamal::Cli::LockError) do
       run_command("deploy")
@@ -78,7 +81,10 @@ class CliMainTest < CliTestCase
     Thread.report_on_exception = false
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*arg| arg[0..1] == [:mkdir, 'kamal_lock-app'] }
+      .with { |*args| args == [ :mkdir, "-p", "kamal" ] }
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |*arg| arg[0..1] == [:mkdir, 'kamal/lock-app'] }
       .raises(SocketError, "getaddrinfo: nodename nor servname provided, or not known")
 
     assert_raises(SSHKit::Runner::ExecuteError) do
@@ -230,7 +236,7 @@ class CliMainTest < CliTestCase
 
   test "audit" do
     run_command("audit").tap do |output|
-      assert_match /tail -n 50 kamal-app-audit.log on 1.1.1.1/, output
+      assert_match %r{tail -n 50 kamal/app-audit.log on 1.1.1.1}, output
       assert_match /App Host: 1.1.1.1/, output
     end
   end
diff --git a/test/cli/server_test.rb b/test/cli/server_test.rb
index 377f6cff..87ff245f 100644
--- a/test/cli/server_test.rb
+++ b/test/cli/server_test.rb
@@ -3,6 +3,7 @@ require_relative "cli_test_case"
 class CliServerTest < CliTestCase
   test "bootstrap already installed" do
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(true).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", "kamal").returns("").at_least_once
 
     assert_equal "", run_command("bootstrap")
   end
@@ -10,6 +11,7 @@ class CliServerTest < CliTestCase
   test "bootstrap install as non-root user" do
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ]', raise_on_non_zero_exit: false).returns(false).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", "kamal").returns("").at_least_once
 
     assert_raise RuntimeError, "Docker is not installed on 1.1.1.1, 1.1.1.3, 1.1.1.4, 1.1.1.2 and can't be automatically installed without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/" do
       run_command("bootstrap")
@@ -20,6 +22,7 @@ class CliServerTest < CliTestCase
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ]', raise_on_non_zero_exit: false).returns(true).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:curl, "-fsSL", "https://get.docker.com", "|", :sh).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", "kamal").returns("").at_least_once
 
     run_command("bootstrap").tap do |output|
       ("1.1.1.1".."1.1.1.4").map do |host|
diff --git a/test/cli/traefik_test.rb b/test/cli/traefik_test.rb
index d5023230..0edaf1f6 100644
--- a/test/cli/traefik_test.rb
+++ b/test/cli/traefik_test.rb
@@ -20,7 +20,7 @@ class CliTraefikTest < CliTestCase
 
   test "reboot --rolling" do
     run_command("reboot", "--rolling").tap do |output|
-      assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.1", output.lines[3]
+      assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.1", output
     end
   end
 
diff --git a/test/commands/auditor_test.rb b/test/commands/auditor_test.rb
index 1ea061fb..1f6f7d0c 100644
--- a/test/commands/auditor_test.rb
+++ b/test/commands/auditor_test.rb
@@ -21,7 +21,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
       :echo,
       "[#{@recorded_at}] [#{@performer}]",
       "app removed container",
-      ">>", "kamal-app-audit.log"
+      ">>", "kamal/app-audit.log"
     ], @auditor.record("app removed container")
   end
 
@@ -31,7 +31,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
         :echo,
         "[#{@recorded_at}] [#{@performer}] [staging]",
         "app removed container",
-        ">>", "kamal-app-staging-audit.log"
+        ">>", "kamal/app-staging-audit.log"
       ], auditor.record("app removed container")
     end
   end
@@ -42,7 +42,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
         :echo,
         "[#{@recorded_at}] [#{@performer}] [web]",
         "app removed container",
-        ">>", "kamal-app-audit.log"
+        ">>", "kamal/app-audit.log"
       ], auditor.record("app removed container")
     end
   end
@@ -52,7 +52,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
       :echo,
       "[#{@recorded_at}] [#{@performer}] [value]",
       "app removed container",
-      ">>", "kamal-app-audit.log"
+      ">>", "kamal/app-audit.log"
     ], @auditor.record("app removed container", detail: "value")
   end
 
diff --git a/test/commands/lock_test.rb b/test/commands/lock_test.rb
index 4ed4fe1f..31d7f790 100644
--- a/test/commands/lock_test.rb
+++ b/test/commands/lock_test.rb
@@ -10,19 +10,19 @@ class CommandsLockTest < ActiveSupport::TestCase
 
   test "status" do
     assert_equal \
-      "stat kamal_lock-app > /dev/null && cat kamal_lock-app/details | base64 -d",
+      "stat kamal/lock-app > /dev/null && cat kamal/lock-app/details | base64 -d",
       new_command.status.join(" ")
   end
 
   test "acquire" do
     assert_match \
-      /mkdir kamal_lock-app && echo ".*" > kamal_lock-app\/details/m,
+      %r{mkdir kamal/lock-app && echo ".*" > kamal/lock-app/details}m,
       new_command.acquire("Hello", "123").join(" ")
   end
 
   test "release" do
     assert_match \
-      "rm kamal_lock-app/details && rm -r kamal_lock-app",
+      "rm kamal/lock-app/details && rm -r kamal/lock-app",
       new_command.release.join(" ")
   end
 
diff --git a/test/commands/server_test.rb b/test/commands/server_test.rb
new file mode 100644
index 00000000..21981b3a
--- /dev/null
+++ b/test/commands/server_test.rb
@@ -0,0 +1,23 @@
+require "test_helper"
+
+class CommandsServerTest < ActiveSupport::TestCase
+  setup do
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
+    }
+  end
+
+  test "ensure run directory" do
+    assert_equal "mkdir -p kamal", new_command.ensure_run_directory.join(" ")
+  end
+
+  test "ensure non default run directory" do
+    assert_equal "mkdir -p /var/run/kamal", new_command(run_directory: "/var/run/kamal").ensure_run_directory.join(" ")
+  end
+
+  private
+    def new_command(extra_config = {})
+      Kamal::Commands::Server.new(Kamal::Configuration.new(@config.merge(extra_config)))
+    end
+end
diff --git a/test/configuration_test.rb b/test/configuration_test.rb
index 425407e5..a660a541 100644
--- a/test/configuration_test.rb
+++ b/test/configuration_test.rb
@@ -283,4 +283,12 @@ class ConfigurationTest < ActiveSupport::TestCase
       Kamal::Configuration.new(@deploy.tap { |c| c.merge!(minimum_version: "10000.0.0") })
     end
   end
+
+  test "run directory" do
+    config = Kamal::Configuration.new(@deploy)
+    assert_equal "kamal", config.run_directory
+
+    config = Kamal::Configuration.new(@deploy.merge!(run_directory: "/root/kamal"))
+    assert_equal "/root/kamal", config.run_directory
+  end
 end

From 787688ea08aaa28cd58765a6f85efb492f8c8711 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Mon, 28 Aug 2023 17:13:52 +0100
Subject: [PATCH 2/6] kamal -> .kamal

---
 lib/kamal/configuration.rb    |  2 +-
 test/cli/cli_test_case.rb     |  6 +++---
 test/cli/main_test.rb         | 12 ++++++------
 test/cli/server_test.rb       |  6 +++---
 test/commands/auditor_test.rb |  8 ++++----
 test/commands/lock_test.rb    |  6 +++---
 test/commands/server_test.rb  |  2 +-
 test/configuration_test.rb    |  2 +-
 8 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb
index af9a0eac..86e137ea 100644
--- a/lib/kamal/configuration.rb
+++ b/lib/kamal/configuration.rb
@@ -58,7 +58,7 @@ class Kamal::Configuration
   end
 
   def run_directory
-    raw_config.run_directory || "kamal"
+    raw_config.run_directory || ".kamal"
   end
 
 
diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb
index 5aa1cc1b..8f99f968 100644
--- a/test/cli/cli_test_case.rb
+++ b/test/cli/cli_test_case.rb
@@ -29,11 +29,11 @@ class CliTestCase < ActiveSupport::TestCase
 
     def stub_setup
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |*args| args == [ :mkdir, "-p", "kamal" ] }
+        .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :mkdir && arg2 == "kamal/lock-app" }
+        .with { |arg1, arg2| arg1 == :mkdir && arg2 == ".kamal/lock-app" }
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :rm && arg2 == "kamal/lock-app/details" }
+        .with { |arg1, arg2| arg1 == :rm && arg2 == ".kamal/lock-app/details" }
     end
 
     def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: nil)
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index d7afce4f..18b31374 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -63,14 +63,14 @@ class CliMainTest < CliTestCase
     Thread.report_on_exception = false
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*args| args == [ :mkdir, "-p", "kamal" ] }
+      .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*arg| arg[0..1] == [:mkdir, 'kamal/lock-app'] }
+      .with { |*arg| arg[0..1] == [:mkdir, ".kamal/lock-app"] }
       .raises(RuntimeError, "mkdir: cannot create directory ‘kamal_lock-app’: File exists")
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_debug)
-      .with(:stat, 'kamal/lock-app', ">", "/dev/null", "&&", :cat, "kamal/lock-app/details", "|", :base64, "-d")
+      .with(:stat, ".kamal/lock-app", ">", "/dev/null", "&&", :cat, ".kamal/lock-app/details", "|", :base64, "-d")
 
     assert_raises(Kamal::Cli::LockError) do
       run_command("deploy")
@@ -81,10 +81,10 @@ class CliMainTest < CliTestCase
     Thread.report_on_exception = false
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*args| args == [ :mkdir, "-p", "kamal" ] }
+      .with { |*args| args == [ :mkdir, "-p", ".kamal" ] }
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*arg| arg[0..1] == [:mkdir, 'kamal/lock-app'] }
+      .with { |*arg| arg[0..1] == [:mkdir, ".kamal/lock-app"] }
       .raises(SocketError, "getaddrinfo: nodename nor servname provided, or not known")
 
     assert_raises(SSHKit::Runner::ExecuteError) do
@@ -236,7 +236,7 @@ class CliMainTest < CliTestCase
 
   test "audit" do
     run_command("audit").tap do |output|
-      assert_match %r{tail -n 50 kamal/app-audit.log on 1.1.1.1}, output
+      assert_match %r{tail -n 50 \.kamal/app-audit.log on 1.1.1.1}, output
       assert_match /App Host: 1.1.1.1/, output
     end
   end
diff --git a/test/cli/server_test.rb b/test/cli/server_test.rb
index 87ff245f..5742d544 100644
--- a/test/cli/server_test.rb
+++ b/test/cli/server_test.rb
@@ -3,7 +3,7 @@ require_relative "cli_test_case"
 class CliServerTest < CliTestCase
   test "bootstrap already installed" do
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(true).at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", "kamal").returns("").at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
 
     assert_equal "", run_command("bootstrap")
   end
@@ -11,7 +11,7 @@ class CliServerTest < CliTestCase
   test "bootstrap install as non-root user" do
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ]', raise_on_non_zero_exit: false).returns(false).at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", "kamal").returns("").at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
 
     assert_raise RuntimeError, "Docker is not installed on 1.1.1.1, 1.1.1.3, 1.1.1.4, 1.1.1.2 and can't be automatically installed without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/" do
       run_command("bootstrap")
@@ -22,7 +22,7 @@ class CliServerTest < CliTestCase
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ]', raise_on_non_zero_exit: false).returns(true).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:curl, "-fsSL", "https://get.docker.com", "|", :sh).at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", "kamal").returns("").at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
 
     run_command("bootstrap").tap do |output|
       ("1.1.1.1".."1.1.1.4").map do |host|
diff --git a/test/commands/auditor_test.rb b/test/commands/auditor_test.rb
index 1f6f7d0c..29b0025d 100644
--- a/test/commands/auditor_test.rb
+++ b/test/commands/auditor_test.rb
@@ -21,7 +21,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
       :echo,
       "[#{@recorded_at}] [#{@performer}]",
       "app removed container",
-      ">>", "kamal/app-audit.log"
+      ">>", ".kamal/app-audit.log"
     ], @auditor.record("app removed container")
   end
 
@@ -31,7 +31,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
         :echo,
         "[#{@recorded_at}] [#{@performer}] [staging]",
         "app removed container",
-        ">>", "kamal/app-staging-audit.log"
+        ">>", ".kamal/app-staging-audit.log"
       ], auditor.record("app removed container")
     end
   end
@@ -42,7 +42,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
         :echo,
         "[#{@recorded_at}] [#{@performer}] [web]",
         "app removed container",
-        ">>", "kamal/app-audit.log"
+        ">>", ".kamal/app-audit.log"
       ], auditor.record("app removed container")
     end
   end
@@ -52,7 +52,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
       :echo,
       "[#{@recorded_at}] [#{@performer}] [value]",
       "app removed container",
-      ">>", "kamal/app-audit.log"
+      ">>", ".kamal/app-audit.log"
     ], @auditor.record("app removed container", detail: "value")
   end
 
diff --git a/test/commands/lock_test.rb b/test/commands/lock_test.rb
index 31d7f790..dc8981f9 100644
--- a/test/commands/lock_test.rb
+++ b/test/commands/lock_test.rb
@@ -10,19 +10,19 @@ class CommandsLockTest < ActiveSupport::TestCase
 
   test "status" do
     assert_equal \
-      "stat kamal/lock-app > /dev/null && cat kamal/lock-app/details | base64 -d",
+      "stat .kamal/lock-app > /dev/null && cat .kamal/lock-app/details | base64 -d",
       new_command.status.join(" ")
   end
 
   test "acquire" do
     assert_match \
-      %r{mkdir kamal/lock-app && echo ".*" > kamal/lock-app/details}m,
+      %r{mkdir \.kamal/lock-app && echo ".*" > \.kamal/lock-app/details}m,
       new_command.acquire("Hello", "123").join(" ")
   end
 
   test "release" do
     assert_match \
-      "rm kamal/lock-app/details && rm -r kamal/lock-app",
+      "rm .kamal/lock-app/details && rm -r .kamal/lock-app",
       new_command.release.join(" ")
   end
 
diff --git a/test/commands/server_test.rb b/test/commands/server_test.rb
index 21981b3a..9e063f77 100644
--- a/test/commands/server_test.rb
+++ b/test/commands/server_test.rb
@@ -9,7 +9,7 @@ class CommandsServerTest < ActiveSupport::TestCase
   end
 
   test "ensure run directory" do
-    assert_equal "mkdir -p kamal", new_command.ensure_run_directory.join(" ")
+    assert_equal "mkdir -p .kamal", new_command.ensure_run_directory.join(" ")
   end
 
   test "ensure non default run directory" do
diff --git a/test/configuration_test.rb b/test/configuration_test.rb
index a660a541..2c0cc95b 100644
--- a/test/configuration_test.rb
+++ b/test/configuration_test.rb
@@ -286,7 +286,7 @@ class ConfigurationTest < ActiveSupport::TestCase
 
   test "run directory" do
     config = Kamal::Configuration.new(@deploy)
-    assert_equal "kamal", config.run_directory
+    assert_equal ".kamal", config.run_directory
 
     config = Kamal::Configuration.new(@deploy.merge!(run_directory: "/root/kamal"))
     assert_equal "/root/kamal", config.run_directory

From bbcc90e4d191af86adf8b833c779e270a5162ccc Mon Sep 17 00:00:00 2001
From: Krzysztof Adamski <krzysztof@adamski.ie>
Date: Tue, 5 Sep 2023 10:53:32 +0200
Subject: [PATCH 3/6] Configurable Healthcheck Expose Port

---
 lib/kamal/commands/healthcheck.rb | 9 ++++++---
 lib/kamal/configuration.rb        | 2 +-
 test/commands/healthcheck_test.rb | 3 ++-
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/kamal/commands/healthcheck.rb b/lib/kamal/commands/healthcheck.rb
index d3c94923..4327087d 100644
--- a/lib/kamal/commands/healthcheck.rb
+++ b/lib/kamal/commands/healthcheck.rb
@@ -1,5 +1,4 @@
 class Kamal::Commands::Healthcheck < Kamal::Commands::Base
-  EXPOSED_PORT = 3999
 
   def run
     web = config.role(:web)
@@ -7,7 +6,7 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
     docker :run,
       "--detach",
       "--name", container_name_with_version,
-      "--publish", "#{EXPOSED_PORT}:#{config.healthcheck["port"]}",
+      "--publish", "#{exposed_port}:#{config.healthcheck["port"]}",
       "--label", "service=#{container_name}",
       "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
       *web.env_args,
@@ -52,6 +51,10 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
     end
 
     def health_url
-      "http://localhost:#{EXPOSED_PORT}#{config.healthcheck["path"]}"
+      "http://localhost:#{exposed_port}#{config.healthcheck["path"]}"
+    end
+
+    def exposed_port
+      config.healthcheck["exposed_port"]
     end
 end
diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb
index d6e8d757..28da91ef 100644
--- a/lib/kamal/configuration.rb
+++ b/lib/kamal/configuration.rb
@@ -145,7 +145,7 @@ class Kamal::Configuration
 
 
   def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7 }.merge(raw_config.healthcheck || {})
+    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999 }.merge(raw_config.healthcheck || {})
   end
 
   def readiness_delay
diff --git a/test/commands/healthcheck_test.rb b/test/commands/healthcheck_test.rb
index 4fad7830..9d47c313 100644
--- a/test/commands/healthcheck_test.rb
+++ b/test/commands/healthcheck_test.rb
@@ -40,8 +40,9 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
 
   test "run with custom options" do
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere" } } }
+    @config[:healthcheck] = { "exposed_port" => 4999 }
     assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 4999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
       new_command.run.join(" ")
   end
 

From c2b2f7ea332361c1e8a531123814b49d77a12aad Mon Sep 17 00:00:00 2001
From: Krzysztof Adamski <krzysztof@adamski.ie>
Date: Wed, 6 Sep 2023 10:16:59 +0200
Subject: [PATCH 4/6] Fixing Tests

---
 test/configuration_test.rb    | 2 +-
 test/integration/main_test.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/configuration_test.rb b/test/configuration_test.rb
index 425407e5..89aeaf84 100644
--- a/test/configuration_test.rb
+++ b/test/configuration_test.rb
@@ -263,7 +263,7 @@ class ConfigurationTest < ActiveSupport::TestCase
         :volume_args=>["--volume", "/local/path:/container/path"],
         :builder=>{},
         :logging=>["--log-opt", "max-size=\"10m\""],
-        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7 }}
+        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999 }}
 
     assert_equal expected_config, @config.to_h
   end
diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb
index 59d73e8f..3a6d5587 100644
--- a/test/integration/main_test.rb
+++ b/test/integration/main_test.rb
@@ -54,6 +54,6 @@ class MainTest < IntegrationTest
     assert_equal({ user: "root", auth_methods: [ "publickey" ], keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])
     assert_equal({ "multiarch" => false, "args" => { "COMMIT_SHA" => version } }, config[:builder])
     assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging]
-    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "cmd" => "wget -qO- http://localhost > /dev/null" }, config[:healthcheck])
+    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cmd" => "wget -qO- http://localhost > /dev/null" }, config[:healthcheck])
   end
 end

From 94bf0906574edd6d73037aad8d8f6e79dc92b3f6 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Wed, 30 Aug 2023 15:16:48 +0100
Subject: [PATCH 5/6] Copy env files to remote hosts

Setting env variables in the docker arguments requires having them on
the deploy host.

Instead we'll add two new commands `kamal env push` and
`kamal env delete` which will manage copying the environment as .env
files to the remote host.

Docker will pick up the file with `--env-file <path-to-file>`. Env files
will be stored under `<kamal run directory>/env`.

Running `kamal env push` will create env files for each role and
accessory, and traefik if required.

`kamal envify` has been updated to also push the env files.

By avoiding using `kamal envify` and creating the local and remote
secrets manually, you can now avoid accessing secrets needed
for the docker runtime environment locally. You will still need build
secrets.

One thing to note - the Docker doesn't parse the environment variables
in the env file, one result of this is that you can't specify multi-line
values - see https://github.com/moby/moby/issues/12997.

We maybe need to look docker config or docker secrets longer term to get
around this.

Hattip to @kevinmcconnell - this was all his idea.
---
 lib/kamal/cli/env.rb                          | 52 ++++++++++++++++
 lib/kamal/cli/main.rb                         |  6 ++
 lib/kamal/commander.rb                        |  4 ++
 lib/kamal/commands/accessory.rb               | 16 ++---
 lib/kamal/commands/app.rb                     |  9 ++-
 lib/kamal/commands/base.rb                    |  8 +++
 lib/kamal/commands/traefik.rb                 | 28 ++++++---
 lib/kamal/configuration.rb                    | 18 ++----
 lib/kamal/configuration/accessory.rb          | 16 ++++-
 lib/kamal/configuration/role.rb               | 16 ++++-
 lib/kamal/utils.rb                            | 34 +++++++---
 test/cli/accessory_test.rb                    |  8 +--
 test/cli/env_test.rb                          | 38 ++++++++++++
 test/cli/healthcheck_test.rb                  |  4 +-
 test/cli/lock_test.rb                         | 10 +--
 test/cli/main_test.rb                         |  6 +-
 test/cli/traefik_test.rb                      |  4 +-
 test/commands/accessory_test.rb               | 20 ++++--
 test/commands/app_test.rb                     | 36 ++++++-----
 test/commands/healthcheck_test.rb             | 10 +--
 test/commands/traefik_test.rb                 | 50 ++++++++++-----
 test/configuration/accessory_test.rb          | 25 +++++---
 test/configuration/role_test.rb               | 55 ++++++++++++----
 test/configuration_test.rb                    | 41 +-----------
 test/integration/accessory_test.rb            |  4 ++
 test/integration/app_test.rb                  |  2 +
 test/integration/docker/deployer/Dockerfile   |  2 +-
 .../docker/deployer/app/config/deploy.yml     |  6 ++
 test/integration/lock_test.rb                 |  2 +
 test/integration/main_test.rb                 | 27 +++++---
 test/integration/traefik_test.rb              |  4 ++
 test/utils_test.rb                            | 62 +++++++++++++++++--
 32 files changed, 453 insertions(+), 170 deletions(-)
 create mode 100644 lib/kamal/cli/env.rb
 create mode 100644 test/cli/env_test.rb

diff --git a/lib/kamal/cli/env.rb b/lib/kamal/cli/env.rb
new file mode 100644
index 00000000..7d4dac10
--- /dev/null
+++ b/lib/kamal/cli/env.rb
@@ -0,0 +1,52 @@
+require "tempfile"
+
+class Kamal::Cli::Env < Kamal::Cli::Base
+  desc "push", "Push the env file to the remote hosts"
+  def push
+    mutating do
+      on(KAMAL.hosts) do
+        KAMAL.roles_on(host).each do |role|
+          role_config = KAMAL.config.role(role)
+          execute *KAMAL.app(role: role).make_env_directory
+          upload! StringIO.new(role_config.env_file), role_config.host_env_file_path, mode: 400
+        end
+      end
+
+      on(KAMAL.traefik_hosts) do
+        execute *KAMAL.traefik.make_env_directory
+        upload! StringIO.new(KAMAL.traefik.env_file), KAMAL.traefik.host_env_file_path, mode: 400
+      end
+
+      on(KAMAL.accessory_hosts) do
+        KAMAL.accessories_on(host).each do |accessory|
+          accessory_config = KAMAL.config.accessory(accessory)
+          execute *KAMAL.accessory(accessory).make_env_directory
+          upload! StringIO.new(accessory_config.env_file), accessory_config.host_env_file_path, mode: 400
+        end
+      end
+    end
+  end
+
+  desc "delete", "Delete the env file from the remote hosts"
+  def delete
+    mutating do
+      on(KAMAL.hosts) do
+        KAMAL.roles_on(host).each do |role|
+          role_config = KAMAL.config.role(role)
+          execute *KAMAL.app(role: role).remove_env_file
+        end
+      end
+
+      on(KAMAL.traefik_hosts) do
+        execute *KAMAL.traefik.remove_env_file
+      end
+
+      on(KAMAL.accessory_hosts) do
+        KAMAL.accessories_on(host).each do |accessory|
+          accessory_config = KAMAL.config.accessory(accessory)
+          execute *KAMAL.accessory(accessory).remove_env_file
+        end
+      end
+    end
+  end
+end
diff --git a/lib/kamal/cli/main.rb b/lib/kamal/cli/main.rb
index 29218d9d..c4154357 100644
--- a/lib/kamal/cli/main.rb
+++ b/lib/kamal/cli/main.rb
@@ -175,6 +175,9 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     end
 
     File.write(env_path, ERB.new(File.read(env_template_path)).result, perm: 0600)
+
+    load_envs # reload new file
+    invoke "kamal:cli:env:push", options
   end
 
   desc "remove", "Remove Traefik, app, accessories, and registry session from servers"
@@ -204,6 +207,9 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   desc "build", "Build application image"
   subcommand "build", Kamal::Cli::Build
 
+  desc "env", "Manage environment files"
+  subcommand "env", Kamal::Cli::Env
+
   desc "healthcheck", "Healthcheck application"
   subcommand "healthcheck", Kamal::Cli::Healthcheck
 
diff --git a/lib/kamal/commander.rb b/lib/kamal/commander.rb
index 47e0cf9f..a98ac2b5 100644
--- a/lib/kamal/commander.rb
+++ b/lib/kamal/commander.rb
@@ -75,6 +75,10 @@ class Kamal::Commander
     config.accessories&.collect(&:name) || []
   end
 
+  def accessories_on(host)
+    config.accessories.select { |accessory| accessory.hosts.include?(host.to_s) }.map(&:name)
+  end
+
 
   def app(role: nil)
     Kamal::Commands::App.new(config, role: role)
diff --git a/lib/kamal/commands/accessory.rb b/lib/kamal/commands/accessory.rb
index 9252e2a2..bd70ac5c 100644
--- a/lib/kamal/commands/accessory.rb
+++ b/lib/kamal/commands/accessory.rb
@@ -86,14 +86,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     end
   end
 
-  def make_directory_for(remote_file)
-    make_directory Pathname.new(remote_file).dirname.to_s
-  end
-
-  def make_directory(path)
-    [ :mkdir, "-p", path ]
-  end
-
   def remove_service_directory
     [ :rm, "-rf", service_name ]
   end
@@ -106,6 +98,14 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     docker :image, :rm, "--force", image
   end
 
+  def make_env_directory
+    make_directory accessory_config.host_env_directory
+  end
+
+  def remove_env_file
+    [:rm, "-f", accessory_config.host_env_file_path]
+  end
+
   private
     def service_filter
       [ "--filter", "label=service=#{service_name}" ]
diff --git a/lib/kamal/commands/app.rb b/lib/kamal/commands/app.rb
index 11d15fcc..cb1f7091 100644
--- a/lib/kamal/commands/app.rb
+++ b/lib/kamal/commands/app.rb
@@ -81,7 +81,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
     docker :run,
       ("-it" if interactive),
       "--rm",
-      *config.env_args,
+      *role&.env_args,
       *config.volume_args,
       *role&.option_args,
       config.absolute_image,
@@ -149,6 +149,13 @@ class Kamal::Commands::App < Kamal::Commands::Base
     docker :tag, config.absolute_image, config.latest_image
   end
 
+  def make_env_directory
+    make_directory config.role(role).host_env_directory
+  end
+
+  def remove_env_file
+    [:rm, "-f", config.role(role).host_env_file_path]
+  end
 
   private
     def container_name(version = nil)
diff --git a/lib/kamal/commands/base.rb b/lib/kamal/commands/base.rb
index 8a413b91..3058df16 100644
--- a/lib/kamal/commands/base.rb
+++ b/lib/kamal/commands/base.rb
@@ -26,6 +26,14 @@ module Kamal::Commands
       docker :container, :ls, *("--all" unless only_running), "--filter", "name=^#{container_name}$", "--quiet"
     end
 
+    def make_directory_for(remote_file)
+      make_directory Pathname.new(remote_file).dirname.to_s
+    end
+
+    def make_directory(path)
+      [ :mkdir, "-p", path ]
+    end
+
     private
       def combine(*commands, by: "&&")
         commands
diff --git a/lib/kamal/commands/traefik.rb b/lib/kamal/commands/traefik.rb
index e77a81de..fbbcea0f 100644
--- a/lib/kamal/commands/traefik.rb
+++ b/lib/kamal/commands/traefik.rb
@@ -1,5 +1,5 @@
 class Kamal::Commands::Traefik < Kamal::Commands::Base
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
 
   DEFAULT_IMAGE = "traefik:v2.9"
   CONTAINER_PORT = 80
@@ -63,6 +63,22 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
     "#{host_port}:#{CONTAINER_PORT}"
   end
 
+  def env_file
+    env_file_with_secrets config.traefik.fetch("env", {})
+  end
+
+  def host_env_file_path
+    File.join host_env_directory, "traefik.env"
+  end
+
+  def make_env_directory
+    make_directory(host_env_directory)
+  end
+
+  def remove_env_file
+    [:rm, "-f", host_env_file_path]
+  end
+
   private
     def publish_args
       argumentize "--publish", port unless config.traefik["publish"] == false
@@ -73,13 +89,11 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
     end
 
     def env_args
-      env_config = config.traefik["env"] || {}
+      argumentize "--env-file", host_env_file_path
+    end
 
-      if env_config.present?
-        argumentize_env_with_secrets(env_config)
-      else
-        []
-      end
+    def host_env_directory
+      File.join config.host_env_directory, "traefik"
     end
 
     def labels
diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb
index 2bb8002b..8db892fc 100644
--- a/lib/kamal/configuration.rb
+++ b/lib/kamal/configuration.rb
@@ -7,7 +7,7 @@ require "net/ssh/proxy/jump"
 
 class Kamal::Configuration
   delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_accessor :destination
   attr_accessor :raw_config
@@ -113,14 +113,6 @@ class Kamal::Configuration
   end
 
 
-  def env_args
-    if raw_config.env.present?
-      argumentize_env_with_secrets(raw_config.env)
-    else
-      []
-    end
-  end
-
   def volume_args
     if raw_config.volumes.present?
       argumentize "--volume", raw_config.volumes
@@ -174,7 +166,6 @@ class Kamal::Configuration
       repository: repository,
       absolute_image: absolute_image,
       service_with_version: service_with_version,
-      env_args: env_args,
       volume_args: volume_args,
       ssh_options: ssh.to_h,
       sshkit: sshkit.to_h,
@@ -199,12 +190,15 @@ class Kamal::Configuration
 
   # Will raise KeyError if any secret ENVs are missing
   def ensure_env_available
-    env_args
-    roles.each(&:env_args)
+    roles.each(&:env_file)
 
     true
   end
 
+  def host_env_directory
+    "#{run_directory}/env"
+  end
+
   private
     # Will raise ArgumentError if any required config keys are missing
     def ensure_required_keys_present
diff --git a/lib/kamal/configuration/accessory.rb b/lib/kamal/configuration/accessory.rb
index 4ffcfc4b..aa5ccfbd 100644
--- a/lib/kamal/configuration/accessory.rb
+++ b/lib/kamal/configuration/accessory.rb
@@ -1,5 +1,5 @@
 class Kamal::Configuration::Accessory
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
 
   attr_accessor :name, :specifics
 
@@ -45,8 +45,20 @@ class Kamal::Configuration::Accessory
     specifics["env"] || {}
   end
 
+  def env_file
+    env_file_with_secrets env
+  end
+
+  def host_env_directory
+    File.join config.host_env_directory, "accessories"
+  end
+
+  def host_env_file_path
+    File.join host_env_directory, "#{service_name}.env"
+  end
+
   def env_args
-    argumentize_env_with_secrets env
+    argumentize "--env-file", host_env_file_path
   end
 
   def files
diff --git a/lib/kamal/configuration/role.rb b/lib/kamal/configuration/role.rb
index 4302430d..f549d459 100644
--- a/lib/kamal/configuration/role.rb
+++ b/lib/kamal/configuration/role.rb
@@ -1,5 +1,5 @@
 class Kamal::Configuration::Role
-  delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
 
   attr_accessor :name
 
@@ -31,8 +31,20 @@ class Kamal::Configuration::Role
     end
   end
 
+  def env_file
+    env_file_with_secrets env
+  end
+
+  def host_env_directory
+    File.join config.host_env_directory, "roles"
+  end
+
+  def host_env_file_path
+    File.join host_env_directory, "#{[config.service, name, config.destination].compact.join("-")}.env"
+  end
+
   def env_args
-    argumentize_env_with_secrets env
+    argumentize "--env-file", host_env_file_path
   end
 
   def health_check_args
diff --git a/lib/kamal/utils.rb b/lib/kamal/utils.rb
index c2461373..6ab1648b 100644
--- a/lib/kamal/utils.rb
+++ b/lib/kamal/utils.rb
@@ -16,14 +16,24 @@ module Kamal::Utils
     end
   end
 
-  # Return a list of shell arguments using the same named argument against the passed attributes,
-  # but redacts and expands secrets.
-  def argumentize_env_with_secrets(env)
-    if (secrets = env["secret"]).present?
-      argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, sensitive: true) + argumentize("-e", env["clear"])
-    else
-      argumentize "-e", env.fetch("clear", env)
-    end
+  def env_file_with_secrets(env)
+    env_file = StringIO.new.tap do |contents|
+      if (secrets = env["secret"]).present?
+        env.fetch("secret", env)&.each do |key|
+          contents << docker_env_file_line(key, ENV.fetch(key))
+        end
+        env["clear"]&.each do |key, value|
+          contents << docker_env_file_line(key, value)
+        end
+      else
+        env.fetch("clear", env)&.each do |key, value|
+          contents << docker_env_file_line(key, value)
+        end
+      end
+    end.string
+
+    # Ensure the file has some contents to avoid the SSHKIT empty file warning
+    env_file || "\n"
   end
 
   # Returns a list of shell-dashed option arguments. If the value is true, it's treated like a value-less option.
@@ -97,4 +107,12 @@ module Kamal::Utils
   def uncommitted_changes
     `git status --porcelain`.strip
   end
+
+  def docker_env_file_line(key, value)
+    if key.include?("\n") || value.to_s.include?("\n")
+      raise ArgumentError, "docker env file format does not support newlines in keys or values, key: #{key}"
+    end
+
+    "#{key.to_s}=#{value.to_s}\n"
+  end
 end
diff --git a/test/cli/accessory_test.rb b/test/cli/accessory_test.rb
index 99dc6e7c..5ecf8038 100644
--- a/test/cli/accessory_test.rb
+++ b/test/cli/accessory_test.rb
@@ -7,7 +7,7 @@ class CliAccessoryTest < CliTestCase
 
     run_command("boot", "mysql").tap do |output|
       assert_match /docker login.*on 1.1.1.3/, output
-      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
     end
   end
 
@@ -21,9 +21,9 @@ class CliAccessoryTest < CliTestCase
       assert_match /docker login.*on 1.1.1.3/, output
       assert_match /docker login.*on 1.1.1.1/, output
       assert_match /docker login.*on 1.1.1.2/, output
-      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
-      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
-      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
     end
   end
 
diff --git a/test/cli/env_test.rb b/test/cli/env_test.rb
new file mode 100644
index 00000000..4d72f558
--- /dev/null
+++ b/test/cli/env_test.rb
@@ -0,0 +1,38 @@
+require_relative "cli_test_case"
+
+class CliEnvTest < CliTestCase
+  test "push" do
+    run_command("push").tap do |output|
+      assert_match "Running /usr/bin/env mkdir -p .kamal/env/roles on 1.1.1.1", output
+      assert_match "Running /usr/bin/env mkdir -p .kamal/env/traefik on 1.1.1.1", output
+      assert_match "Running /usr/bin/env mkdir -p .kamal/env/accessories on 1.1.1.1", output
+      assert_match "Running /usr/bin/env mkdir -p .kamal/env/roles on 1.1.1.1", output
+      assert_match "Running /usr/bin/env mkdir -p .kamal/env/traefik on 1.1.1.2", output
+      assert_match "Running /usr/bin/env mkdir -p .kamal/env/accessories on 1.1.1.1", output
+      assert_match ".kamal/env/roles/app-web.env", output
+      assert_match ".kamal/env/roles/app-workers.env", output
+      assert_match ".kamal/env/traefik/traefik.env", output
+      assert_match ".kamal/env/accessories/app-redis.env", output
+
+    end
+  end
+
+  test "delete" do
+    run_command("delete").tap do |output|
+      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-web.env on 1.1.1.1", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-web.env on 1.1.1.2", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-workers.env on 1.1.1.3", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-workers.env on 1.1.1.4", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/traefik/traefik.env on 1.1.1.1", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/traefik/traefik.env on 1.1.1.2", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-redis.env on 1.1.1.1", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-redis.env on 1.1.1.2", output
+      assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-mysql.env on 1.1.1.3", output
+    end
+  end
+
+  private
+    def run_command(*command)
+      stdouted { Kamal::Cli::Env.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+    end
+end
diff --git a/test/cli/healthcheck_test.rb b/test/cli/healthcheck_test.rb
index 9cef1c8f..f9c3aa9c 100644
--- a/test/cli/healthcheck_test.rb
+++ b/test/cli/healthcheck_test.rb
@@ -10,7 +10,7 @@ class CliHealthcheckTest < CliTestCase
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
+      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--env-file", ".kamal/env/roles/app-web.env", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false)
 
@@ -39,7 +39,7 @@ class CliHealthcheckTest < CliTestCase
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
+      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "KAMAL_CONTAINER_NAME=\"healthcheck-app\"", "--env-file", ".kamal/env/roles/app-web.env", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false)
 
diff --git a/test/cli/lock_test.rb b/test/cli/lock_test.rb
index 9521480f..8972bdc5 100644
--- a/test/cli/lock_test.rb
+++ b/test/cli/lock_test.rb
@@ -2,19 +2,19 @@ require_relative "cli_test_case"
 
 class CliLockTest < CliTestCase
   test "status" do
-    run_command("status") do |output|
-      assert_match "stat lock", output
+    run_command("status").tap do |output|
+      assert_match "Running /usr/bin/env stat .kamal/lock-app > /dev/null && cat .kamal/lock-app/details | base64 -d on 1.1.1.1", output
     end
   end
 
   test "release" do
-    run_command("release") do |output|
-      assert_match "rm -rf lock", output
+    run_command("release").tap do |output|
+      assert_match "Released the deploy lock", output
     end
   end
 
   private
     def run_command(*command)
-      stdouted { Kamal::Cli::Lock.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Lock.start([*command, "-v", "-c", "test/fixtures/deploy_with_accessories.yml"]) }
     end
 end
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index 18b31374..04f11575 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -339,10 +339,10 @@ class CliMainTest < CliTestCase
   end
 
   test "envify with destination" do
-    File.expects(:read).with(".env.staging.erb").returns("HELLO=<%= 'world' %>")
-    File.expects(:write).with(".env.staging", "HELLO=world", perm: 0600)
+    File.expects(:read).with(".env.world.erb").returns("HELLO=<%= 'world' %>")
+    File.expects(:write).with(".env.world", "HELLO=world", perm: 0600)
 
-    run_command("envify", "-d", "staging")
+    run_command("envify", "-d", "world", config_file: "deploy_for_dest")
   end
 
   test "remove with confirmation" do
diff --git a/test/cli/traefik_test.rb b/test/cli/traefik_test.rb
index 0edaf1f6..1b19b033 100644
--- a/test/cli/traefik_test.rb
+++ b/test/cli/traefik_test.rb
@@ -4,7 +4,7 @@ class CliTraefikTest < CliTestCase
   test "boot" do
     run_command("boot").tap do |output|
       assert_match "docker login", output
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
+      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
     end
   end
 
@@ -14,7 +14,7 @@ class CliTraefikTest < CliTestCase
     run_command("reboot").tap do |output|
       assert_match "docker container stop traefik", output
       assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik", output
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
+      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
     end
   end
 
diff --git a/test/commands/accessory_test.rb b/test/commands/accessory_test.rb
index 89279ccc..2825f3de 100644
--- a/test/commands/accessory_test.rb
+++ b/test/commands/accessory_test.rb
@@ -49,15 +49,15 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 -e MYSQL_ROOT_PASSWORD=\"secret123\" -e MYSQL_ROOT_HOST=\"%\" --label service=\"app-mysql\" private.registry/mysql:8.0",
+      "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --label service=\"app-mysql\" private.registry/mysql:8.0",
       new_command(:mysql).run.join(" ")
 
     assert_equal \
-      "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 -e SOMETHING=\"else\" --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest",
+      "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest",
       new_command(:redis).run.join(" ")
 
     assert_equal \
-      "docker run --name app-busybox --detach --restart unless-stopped --log-opt max-size=\"10m\" --label service=\"app-busybox\" busybox:latest",
+      "docker run --name app-busybox --detach --restart unless-stopped --log-opt max-size=\"10m\" --env-file .kamal/env/accessories/app-busybox.env --label service=\"app-busybox\" busybox:latest",
       new_command(:busybox).run.join(" ")
   end
 
@@ -65,7 +65,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --name app-busybox --detach --restart unless-stopped --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app-busybox\" busybox:latest",
+      "docker run --name app-busybox --detach --restart unless-stopped --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --env-file .kamal/env/accessories/app-busybox.env --label service=\"app-busybox\" busybox:latest",
       new_command(:busybox).run.join(" ")
   end
 
@@ -90,7 +90,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "execute in new container" do
     assert_equal \
-      "docker run --rm -e MYSQL_ROOT_PASSWORD=\"secret123\" -e MYSQL_ROOT_HOST=\"%\" private.registry/mysql:8.0 mysql -u root",
+      "docker run --rm --env-file .kamal/env/accessories/app-mysql.env private.registry/mysql:8.0 mysql -u root",
       new_command(:mysql).execute_in_new_container("mysql", "-u", "root").join(" ")
   end
 
@@ -102,7 +102,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
 
   test "execute in new container over ssh" do
     new_command(:mysql).stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do
-      assert_match %r|docker run -it --rm -e MYSQL_ROOT_PASSWORD=\"secret123\" -e MYSQL_ROOT_HOST=\"%\" private.registry/mysql:8.0 mysql -u root|,
+      assert_match %r|docker run -it --rm --env-file .kamal/env/accessories/app-mysql.env private.registry/mysql:8.0 mysql -u root|,
         new_command(:mysql).execute_in_new_container_over_ssh("mysql", "-u", "root")
     end
   end
@@ -144,6 +144,14 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
       new_command(:mysql).remove_image.join(" ")
   end
 
+  test "make_env_directory" do
+    assert_equal "mkdir -p .kamal/env/accessories", new_command(:mysql).make_env_directory.join(" ")
+  end
+
+  test "remove_env_file" do
+    assert_equal "rm -f .kamal/env/accessories/app-mysql.env", new_command(:mysql).remove_env_file.join(" ")
+  end
+
   private
     def new_command(accessory)
       Kamal::Commands::Accessory.new(Kamal::Configuration.new(@config), name: accessory)
diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb
index 69060afe..ab3f6ace 100644
--- a/test/commands/app_test.rb
+++ b/test/commands/app_test.rb
@@ -13,13 +13,13 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with hostname" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run(hostname: "myhost").join(" ")
   end
 
@@ -27,7 +27,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:volumes] = ["/local/path:/container/path" ]
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -35,7 +35,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "path" => "/healthz" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/healthz || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/healthz || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -43,7 +43,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "cmd" => "/bin/up" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"/bin/up\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/up\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -51,14 +51,14 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"/bin/healthy\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/healthy\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with custom options" do
     @config[:servers] = { "web" => [ "1.1.1.1" ], "jobs" => { "hosts" => [ "1.1.1.2" ], "cmd" => "bin/jobs", "options" => { "mount" => "somewhere", "cap-add" => true } } }
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
+      "docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" --env-file .kamal/env/roles/app-jobs.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
       new_command(role: "jobs").run.join(" ")
   end
 
@@ -66,7 +66,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -85,13 +85,13 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "start_or_run" do
     assert_equal \
-      "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.start_or_run.join(" ")
   end
 
   test "start_or_run with hostname" do
     assert_equal \
-      "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.start_or_run(hostname: "myhost").join(" ")
   end
 
@@ -167,14 +167,14 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "execute in new container" do
     assert_equal \
-      "docker run --rm -e RAILS_MASTER_KEY=\"456\" dhh/app:999 bin/rails db:setup",
+      "docker run --rm --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails db:setup",
       new_command.execute_in_new_container("bin/rails", "db:setup").join(" ")
   end
 
   test "execute in new container with custom options" do
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } }
     assert_equal \
-      "docker run --rm -e RAILS_MASTER_KEY=\"456\" --mount \"somewhere\" --cap-add dhh/app:999 bin/rails db:setup",
+      "docker run --rm --env-file .kamal/env/roles/app-web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails db:setup",
       new_command.execute_in_new_container("bin/rails", "db:setup").join(" ")
   end
 
@@ -185,13 +185,13 @@ class CommandsAppTest < ActiveSupport::TestCase
   end
 
   test "execute in new container over ssh" do
-    assert_match %r|docker run -it --rm -e RAILS_MASTER_KEY=\"456\" dhh/app:999 bin/rails c|,
+    assert_match %r|docker run -it --rm --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails c|,
       new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1")
   end
 
   test "execute in new container with custom options over ssh" do
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } }
-    assert_match %r|docker run -it --rm -e RAILS_MASTER_KEY=\"456\" --mount \"somewhere\" --cap-add dhh/app:999 bin/rails c|,
+    assert_match %r|docker run -it --rm --env-file .kamal/env/roles/app-web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails c|,
       new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1")
   end
 
@@ -334,6 +334,14 @@ class CommandsAppTest < ActiveSupport::TestCase
       new_command.tag_current_as_latest.join(" ")
   end
 
+  test "make_env_directory" do
+    assert_equal "mkdir -p .kamal/env/roles", new_command.make_env_directory.join(" ")
+  end
+
+  test "remove_env_file" do
+    assert_equal "rm -f .kamal/env/roles/app-web.env", new_command.remove_env_file.join(" ")
+  end
+
   private
     def new_command(role: "web")
       Kamal::Commands::App.new(Kamal::Configuration.new(@config, destination: @destination, version: "999"), role: role)
diff --git a/test/commands/healthcheck_test.rb b/test/commands/healthcheck_test.rb
index 9d47c313..5ef7fc03 100644
--- a/test/commands/healthcheck_test.rb
+++ b/test/commands/healthcheck_test.rb
@@ -10,7 +10,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
       new_command.run.join(" ")
   end
 
@@ -18,7 +18,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "port" => 3001 }
 
     assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3001 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"curl -f http://localhost:3001/up || exit 1\" --health-interval \"1s\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 3999:3001 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3001/up || exit 1\" --health-interval \"1s\" dhh/app:123",
       new_command.run.join(" ")
   end
 
@@ -26,7 +26,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
     @destination = "staging"
 
     assert_equal \
-      "docker run --detach --name healthcheck-app-staging-123 --publish 3999:3000 --label service=healthcheck-app-staging -e KAMAL_CONTAINER_NAME=\"healthcheck-app-staging\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-staging-123 --publish 3999:3000 --label service=healthcheck-app-staging -e KAMAL_CONTAINER_NAME=\"healthcheck-app-staging\" --env-file .kamal/env/roles/app-web-staging.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
       new_command.run.join(" ")
   end
 
@@ -34,7 +34,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "cmd" => "/bin/up" }
 
     assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"/bin/up\" --health-interval \"1s\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/up\" --health-interval \"1s\" dhh/app:123",
       new_command.run.join(" ")
   end
 
@@ -42,7 +42,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere" } } }
     @config[:healthcheck] = { "exposed_port" => 4999 }
     assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 4999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 4999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
       new_command.run.join(" ")
   end
 
diff --git a/test/commands/traefik_test.rb b/test/commands/traefik_test.rb
index b6b47bfd..5a651dee 100644
--- a/test/commands/traefik_test.rb
+++ b/test/commands/traefik_test.rb
@@ -18,72 +18,72 @@ class CommandsTraefikTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["host_port"] = "8080"
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["publish"] = false
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with ports configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["options"] = {"publish" => %w[9000:9000 9001:9001]}
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with volumes configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json] }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with several options configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json], "publish" => %w[8080:8080], "memory" => "512m"}
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with labels configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["labels"] = { "traefik.http.routers.dashboard.service" => "api@internal", "traefik.http.routers.dashboard.middlewares" => "auth" }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --label traefik.http.routers.dashboard.service=\"api@internal\" --label traefik.http.routers.dashboard.middlewares=\"auth\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.dashboard.service=\"api@internal\" --label traefik.http.routers.dashboard.middlewares=\"auth\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
   test "run with env configured" do
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
 
     @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock -e EXAMPLE_API_KEY=\"456\" --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -91,7 +91,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
     @config.delete(:traefik)
 
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{Kamal::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"",
       new_command.run.join(" ")
   end
 
@@ -99,7 +99,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -107,7 +107,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
     @config[:traefik]["args"]["log.level"] = "ERROR"
 
     assert_equal \
-    "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"ERROR\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+    "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=\"ERROR\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
       new_command.run.join(" ")
   end
 
@@ -177,6 +177,24 @@ class CommandsTraefikTest < ActiveSupport::TestCase
       new_command.follow_logs(host: @config[:servers].first, grep: 'hello!')
   end
 
+  test "env_file" do
+    @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
+
+    assert_equal "EXAMPLE_API_KEY=456\n", new_command.env_file
+  end
+
+  test "host_env_file_path" do
+    assert_equal ".kamal/env/traefik/traefik.env", new_command.host_env_file_path
+  end
+
+  test "make_env_directory" do
+    assert_equal "mkdir -p .kamal/env/traefik", new_command.make_env_directory.join(" ")
+  end
+
+  test "remove_env_file" do
+    assert_equal "rm -f .kamal/env/traefik/traefik.env", new_command.remove_env_file.join(" ")
+  end
+
   private
     def new_command
       Kamal::Commands::Traefik.new(Kamal::Configuration.new(@config, version: "123"))
diff --git a/test/configuration/accessory_test.rb b/test/configuration/accessory_test.rb
index 4f962304..73136239 100644
--- a/test/configuration/accessory_test.rb
+++ b/test/configuration/accessory_test.rb
@@ -110,19 +110,30 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
     assert_equal ["--label", "service=\"app-redis\"", "--label", "cache=\"true\""], @config.accessory(:redis).label_args
   end
 
-  test "env args with secret" do
+  test "env args" do
+    assert_equal ["--env-file", ".kamal/env/accessories/app-mysql.env"], @config.accessory(:mysql).env_args
+    assert_equal ["--env-file", ".kamal/env/accessories/app-redis.env"], @config.accessory(:redis).env_args
+  end
+
+  test "env file with secret" do
     ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
 
-    @config.accessory(:mysql).env_args.tap do |env_args|
-      assert_equal ["-e", "MYSQL_ROOT_PASSWORD=\"secret123\"", "-e", "MYSQL_ROOT_HOST=\"%\""], Kamal::Utils.unredacted(env_args)
-      assert_equal ["-e", "MYSQL_ROOT_PASSWORD=[REDACTED]", "-e", "MYSQL_ROOT_HOST=\"%\""], Kamal::Utils.redacted(env_args)
-    end
+    expected = <<~ENV
+      MYSQL_ROOT_PASSWORD=secret123
+      MYSQL_ROOT_HOST=%
+    ENV
+
+    assert_equal expected, @config.accessory(:mysql).env_file
   ensure
     ENV["MYSQL_ROOT_PASSWORD"] = nil
   end
 
-  test "env args without secret" do
-    assert_equal ["-e", "SOMETHING=\"else\""], @config.accessory(:redis).env_args
+  test "host_env_directory" do
+    assert_equal ".kamal/env/accessories", @config.accessory(:mysql).host_env_directory
+  end
+
+  test "host_env_file_path" do
+    assert_equal ".kamal/env/accessories/app-mysql.env", @config.accessory(:mysql).host_env_file_path
   end
 
   test "volume args" do
diff --git a/test/configuration/role_test.rb b/test/configuration/role_test.rb
index c554965d..b29ac2b5 100644
--- a/test/configuration/role_test.rb
+++ b/test/configuration/role_test.rb
@@ -71,7 +71,17 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
 
   test "env overwritten by role" do
     assert_equal "redis://a/b", @config_with_roles.role(:workers).env["REDIS_URL"]
-    assert_equal ["-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], @config_with_roles.role(:workers).env_args
+
+    expected_env = <<~ENV
+      REDIS_URL=redis://a/b
+      WEB_CONCURRENCY=4
+    ENV
+
+    assert_equal expected_env, @config_with_roles.role(:workers).env_file
+  end
+
+  test "env args" do
+    assert_equal ["--env-file", ".kamal/env/roles/app-workers.env"], @config_with_roles.role(:workers).env_args
   end
 
   test "env secret overwritten by role" do
@@ -97,10 +107,14 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     ENV["REDIS_PASSWORD"] = "secret456"
     ENV["DB_PASSWORD"] = "secret&\"123"
 
-    @config_with_roles.role(:workers).env_args.tap do |env_args|
-      assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "DB_PASSWORD=\"secret&\\\"123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Kamal::Utils.unredacted(env_args)
-      assert_equal ["-e", "REDIS_PASSWORD=[REDACTED]", "-e", "DB_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Kamal::Utils.redacted(env_args)
-    end
+    expected = <<~ENV
+      REDIS_PASSWORD=secret456
+      DB_PASSWORD=secret&\"123
+      REDIS_URL=redis://a/b
+      WEB_CONCURRENCY=4
+    ENV
+
+    assert_equal expected, @config_with_roles.role(:workers).env_file
   ensure
     ENV["REDIS_PASSWORD"] = nil
     ENV["DB_PASSWORD"] = nil
@@ -119,10 +133,13 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
 
     ENV["DB_PASSWORD"] = "secret123"
 
-    @config_with_roles.role(:workers).env_args.tap do |env_args|
-      assert_equal ["-e", "DB_PASSWORD=\"secret123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Kamal::Utils.unredacted(env_args)
-      assert_equal ["-e", "DB_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Kamal::Utils.redacted(env_args)
-    end
+    expected = <<~ENV
+      DB_PASSWORD=secret123
+      REDIS_URL=redis://a/b
+      WEB_CONCURRENCY=4
+    ENV
+
+    assert_equal expected, @config_with_roles.role(:workers).env_file
   ensure
     ENV["DB_PASSWORD"] = nil
   end
@@ -139,11 +156,23 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
 
     ENV["REDIS_PASSWORD"] = "secret456"
 
-    @config_with_roles.role(:workers).env_args.tap do |env_args|
-      assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Kamal::Utils.unredacted(env_args)
-      assert_equal ["-e", "REDIS_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Kamal::Utils.redacted(env_args)
-    end
+    expected = <<~ENV
+      REDIS_PASSWORD=secret456
+      REDIS_URL=redis://a/b
+      WEB_CONCURRENCY=4
+    ENV
+
+    assert_equal expected, @config_with_roles.role(:workers).env_file
   ensure
     ENV["REDIS_PASSWORD"] = nil
   end
+
+  test "host_env_directory" do
+    assert_equal ".kamal/env/roles", @config_with_roles.role(:workers).host_env_directory
+  end
+
+  test "host_env_file_path" do
+    assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).host_env_file_path
+  end
+
 end
diff --git a/test/configuration_test.rb b/test/configuration_test.rb
index 61505312..d31e0dbd 100644
--- a/test/configuration_test.rb
+++ b/test/configuration_test.rb
@@ -124,45 +124,7 @@ class ConfigurationTest < ActiveSupport::TestCase
     assert_equal "app-missing", @config.service_with_version
   end
 
-  test "env args" do
-    assert_equal [ "-e", "REDIS_URL=\"redis://x/y\"" ], @config.env_args
-  end
-
-  test "env args with clear and secrets" do
-    ENV["PASSWORD"] = "secret123"
-
-    config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!({
-      env: { "clear" => { "PORT" => "3000" }, "secret" => [ "PASSWORD" ] }
-    }) })
-
-    assert_equal [ "-e", "PASSWORD=\"secret123\"", "-e", "PORT=\"3000\"" ], Kamal::Utils.unredacted(config.env_args)
-    assert_equal [ "-e", "PASSWORD=[REDACTED]", "-e", "PORT=\"3000\"" ], Kamal::Utils.redacted(config.env_args)
-  ensure
-    ENV["PASSWORD"] = nil
-  end
-
-  test "env args with only clear" do
-    config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!({
-      env: { "clear" => { "PORT" => "3000" } }
-    }) })
-
-    assert_equal [ "-e", "PORT=\"3000\"" ], config.env_args
-  end
-
-  test "env args with only secrets" do
-    ENV["PASSWORD"] = "secret123"
-
-    config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!({
-      env: { "secret" => [ "PASSWORD" ] }
-    }) })
-
-    assert_equal [ "-e", "PASSWORD=\"secret123\"" ], Kamal::Utils.unredacted(config.env_args)
-    assert_equal [ "-e", "PASSWORD=[REDACTED]" ], Kamal::Utils.redacted(config.env_args)
-  ensure
-    ENV["PASSWORD"] = nil
-  end
-
-  test "env args with missing secret" do
+  test "env with missing secret" do
     assert_raises(KeyError) do
       config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!({
         env: { "secret" => [ "PASSWORD" ] }
@@ -257,7 +219,6 @@ class ConfigurationTest < ActiveSupport::TestCase
         :repository=>"dhh/app",
         :absolute_image=>"dhh/app:missing",
         :service_with_version=>"app-missing",
-        :env_args=>["-e", "REDIS_URL=\"redis://x/y\""],
         :ssh_options=>{ :user=>"root", :auth_methods=>["publickey"], log_level: :fatal, keepalive: true, keepalive_interval: 30 },
         :sshkit=>{},
         :volume_args=>["--volume", "/local/path:/container/path"],
diff --git a/test/integration/accessory_test.rb b/test/integration/accessory_test.rb
index 102c02d3..87ae6de8 100644
--- a/test/integration/accessory_test.rb
+++ b/test/integration/accessory_test.rb
@@ -2,6 +2,8 @@ require_relative "integration_test"
 
 class AccessoryTest < IntegrationTest
   test "boot, stop, start, restart, logs, remove" do
+    kamal :envify
+
     kamal :accessory, :boot, :busybox
     assert_accessory_running :busybox
 
@@ -19,6 +21,8 @@ class AccessoryTest < IntegrationTest
 
     kamal :accessory, :remove, :busybox, "-y"
     assert_accessory_not_running :busybox
+
+    kamal :env, :delete
   end
 
   private
diff --git a/test/integration/app_test.rb b/test/integration/app_test.rb
index 1f5c5b1c..37a4dd71 100644
--- a/test/integration/app_test.rb
+++ b/test/integration/app_test.rb
@@ -2,6 +2,8 @@ require_relative "integration_test"
 
 class AppTest < IntegrationTest
   test "stop, start, boot, logs, images, containers, exec, remove" do
+    kamal :envify
+
     kamal :deploy
 
     assert_app_is_up
diff --git a/test/integration/docker/deployer/Dockerfile b/test/integration/docker/deployer/Dockerfile
index 11900139..b964e0e3 100644
--- a/test/integration/docker/deployer/Dockerfile
+++ b/test/integration/docker/deployer/Dockerfile
@@ -23,7 +23,7 @@ RUN mkdir -p /etc/docker/certs.d/registry:4443 && ln -s /shared/certs/domain.crt
 
 RUN git config --global user.email "deployer@example.com"
 RUN git config --global user.name "Deployer"
-RUN git init && git add . && git commit -am "Initial version"
+RUN git init && echo ".env" >> .gitignore && git add . && git commit -am "Initial version"
 
 HEALTHCHECK --interval=1s CMD pgrep sleep
 
diff --git a/test/integration/docker/deployer/app/config/deploy.yml b/test/integration/docker/deployer/app/config/deploy.yml
index bc2cb28c..6ecb94b3 100644
--- a/test/integration/docker/deployer/app/config/deploy.yml
+++ b/test/integration/docker/deployer/app/config/deploy.yml
@@ -3,6 +3,12 @@ image: app
 servers:
   - vm1
   - vm2
+env:
+  clear:
+    CLEAR_TOKEN: '4321'
+  secret:
+    - SECRET_TOKEN
+
 registry:
   server: registry:4443
   username: root
diff --git a/test/integration/lock_test.rb b/test/integration/lock_test.rb
index 22795b37..c9d88a91 100644
--- a/test/integration/lock_test.rb
+++ b/test/integration/lock_test.rb
@@ -2,6 +2,8 @@ require_relative "integration_test"
 
 class LockTest < IntegrationTest
   test "acquire, release, status" do
+    kamal :envify
+
     kamal :lock, :acquire, "-m 'Integration Tests'"
 
     status = kamal :lock, :status, capture: true
diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb
index 3a6d5587..08e2567f 100644
--- a/test/integration/main_test.rb
+++ b/test/integration/main_test.rb
@@ -1,7 +1,11 @@
 require_relative "integration_test"
 
 class MainTest < IntegrationTest
-  test "deploy, redeploy, rollback, details and audit" do
+  test "envify, deploy, redeploy, rollback, details and audit" do
+    kamal :envify
+    assert_local_env_file "SECRET_TOKEN=1234"
+    assert_remote_env_file "SECRET_TOKEN=1234\nCLEAR_TOKEN=4321"
+
     first_version = latest_app_version
 
     assert_app_is_down
@@ -30,12 +34,9 @@ class MainTest < IntegrationTest
 
     audit = kamal :audit, capture: true
     assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
-  end
 
-  test "envify" do
-    kamal :envify
-
-    assert_equal "SECRET_TOKEN=1234", deployer_exec("cat .env", capture: true)
+    kamal :env, :delete
+    assert_no_remote_env_file
   end
 
   test "config" do
@@ -49,11 +50,23 @@ class MainTest < IntegrationTest
     assert_equal "registry:4443/app", config[:repository]
     assert_equal "registry:4443/app:#{version}", config[:absolute_image]
     assert_equal "app-#{version}", config[:service_with_version]
-    assert_equal [], config[:env_args]
     assert_equal [], config[:volume_args]
     assert_equal({ user: "root", auth_methods: [ "publickey" ], keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])
     assert_equal({ "multiarch" => false, "args" => { "COMMIT_SHA" => version } }, config[:builder])
     assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging]
     assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cmd" => "wget -qO- http://localhost > /dev/null" }, config[:healthcheck])
   end
+
+  private
+    def assert_local_env_file(contents)
+      assert_equal contents, deployer_exec("cat .env", capture: true)
+    end
+
+    def assert_remote_env_file(contents)
+      assert_equal contents, docker_compose("exec vm1 cat /root/.kamal/env/roles/app-web.env", capture: true)
+    end
+
+    def assert_no_remote_env_file
+      assert_equal "nofile", docker_compose("exec vm1 stat /root/.kamal/env/roles/app-web.env 2> /dev/null || echo nofile", capture: true)
+    end
 end
diff --git a/test/integration/traefik_test.rb b/test/integration/traefik_test.rb
index 45a08e22..92211c43 100644
--- a/test/integration/traefik_test.rb
+++ b/test/integration/traefik_test.rb
@@ -2,6 +2,8 @@ require_relative "integration_test"
 
 class TraefikTest < IntegrationTest
   test "boot, reboot, stop, start, restart, logs, remove" do
+    kamal :envify
+
     kamal :traefik, :boot
     assert_traefik_running
 
@@ -33,6 +35,8 @@ class TraefikTest < IntegrationTest
 
     kamal :traefik, :remove
     assert_traefik_not_running
+
+    kamal :env, :delete
   end
 
   private
diff --git a/test/utils_test.rb b/test/utils_test.rb
index 3cc4af76..cef0a7fa 100644
--- a/test/utils_test.rb
+++ b/test/utils_test.rb
@@ -11,13 +11,65 @@ class UtilsTest < ActiveSupport::TestCase
       Kamal::Utils.argumentize("--label", { foo: "bar" }, sensitive: true).last
   end
 
-  test "argumentize_env_with_secrets" do
-    ENV.expects(:fetch).with("FOO").returns("secret")
+  test "env file simple" do
+    env = {
+      "foo" => "bar",
+      "baz" => "haz"
+    }
 
-    args = Kamal::Utils.argumentize_env_with_secrets({ "secret" => [ "FOO" ], "clear" => { BAZ: "qux" } })
+    assert_equal "foo=bar\nbaz=haz\n", \
+      Kamal::Utils.env_file_with_secrets(env)
+  end
 
-    assert_equal [ "-e", "FOO=[REDACTED]", "-e", "BAZ=\"qux\"" ], Kamal::Utils.redacted(args)
-    assert_equal [ "-e", "FOO=\"secret\"", "-e", "BAZ=\"qux\"" ], Kamal::Utils.unredacted(args)
+  test "env file clear" do
+    env = {
+      "clear" => {
+        "foo" => "bar",
+        "baz" => "haz"
+      }
+    }
+
+    assert_equal "foo=bar\nbaz=haz\n", \
+      Kamal::Utils.env_file_with_secrets(env)
+  end
+
+  test "env file secret" do
+    ENV["PASSWORD"] = "hello"
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_equal "PASSWORD=hello\n", \
+      Kamal::Utils.env_file_with_secrets(env)
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "env file missing secret" do
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_raises(KeyError) { Kamal::Utils.env_file_with_secrets(env) }
+
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "env file secret and clear" do
+    ENV["PASSWORD"] = "hello"
+    env = {
+      "secret" => [ "PASSWORD" ],
+      "clear" => {
+        "foo" => "bar",
+        "baz" => "haz"
+      }
+    }
+
+    assert_equal "PASSWORD=hello\nfoo=bar\nbaz=haz\n", \
+      Kamal::Utils.env_file_with_secrets(env)
+  ensure
+    ENV.delete "PASSWORD"
   end
 
   test "optionize" do

From 8a41d15b6972a8dacb2758d3fb7af55ba2a260a7 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Thu, 31 Aug 2023 10:21:57 +0100
Subject: [PATCH 6/6] Zero downtime deployment with cord file

When replacing a container currently we:
1. Boot the new container
2. Wait for it to become healthy
3. Stop the old container

Traefik will send requests to the old container until it notices that it
is unhealthy. But it may have stopped serving requests before that point
which can result in errors.

To get round that the new boot process is:

1. Create a directory with a single file on the host
2. Boot the new container, mounting the cord file into /tmp and
including a check for the file in the docker healthcheck
3. Wait for it to become healthy
4. Delete the healthcheck file ("cut the cord") for the old container
5. Wait for it to become unhealthy and give Traefik a couple of seconds
to notice
6. Stop the old container

The extra steps ensure that Traefik stops sending requests before the
old container is shutdown.
---
 lib/kamal/cli/app.rb                          | 21 ++++++-
 lib/kamal/commands/app.rb                     | 47 +++++++++-------
 lib/kamal/commands/base.rb                    | 10 ++++
 lib/kamal/commands/healthcheck.rb             |  2 +-
 lib/kamal/configuration.rb                    | 14 ++++-
 lib/kamal/configuration/role.rb               | 55 ++++++++++++++++---
 lib/kamal/utils/healthcheck_poller.rb         | 29 +++++++++-
 test/cli/app_test.rb                          | 17 +++++-
 test/cli/healthcheck_test.rb                  |  1 +
 test/cli/main_test.rb                         | 19 +++++--
 test/cli/traefik_test.rb                      |  2 +
 test/commands/app_test.rb                     | 41 +++++++-------
 test/configuration/role_test.rb               | 20 +++++++
 test/configuration_test.rb                    | 15 ++++-
 test/integration/docker/deployer/Dockerfile   |  2 +
 .../docker/deployer/app/config/deploy.yml     |  2 +-
 test/integration/main_test.rb                 |  2 +-
 17 files changed, 234 insertions(+), 65 deletions(-)

diff --git a/lib/kamal/cli/app.rb b/lib/kamal/cli/app.rb
index 4892acd2..9c34e8dd 100644
--- a/lib/kamal/cli/app.rb
+++ b/lib/kamal/cli/app.rb
@@ -18,8 +18,9 @@ class Kamal::Cli::App < Kamal::Cli::Base
             roles.each do |role|
               app = KAMAL.app(role: role)
               auditor = KAMAL.auditor(role: role)
+              role_config = KAMAL.config.role(role)
 
-              if capture_with_info(*app.container_id_for_version(version, only_running: true), raise_on_non_zero_exit: false).present?
+              if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
                 tmp_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
                 info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
                 execute *auditor.record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
@@ -29,11 +30,25 @@ class Kamal::Cli::App < Kamal::Cli::Base
               execute *auditor.record("Booted app version #{version}"), verbosity: :debug
 
               old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-              execute *app.start_or_run(hostname: "#{host}-#{SecureRandom.hex(6)}")
+
+              if role_config.uses_cord?
+                execute *app.tie_cord(role_config.cord_host_file)
+              end
+
+              execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
 
               Kamal::Utils::HealthcheckPoller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
 
-              execute *app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
+              if old_version.present?
+                if role_config.uses_cord?
+                  cord = capture_with_info(*app.cord(version: old_version), raise_on_non_zero_exit: false).strip
+                  if cord.present?
+                    execute *app.cut_cord(cord)
+                    Kamal::Utils::HealthcheckPoller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
+                  end
+                end
+                execute *app.stop(version: old_version), raise_on_non_zero_exit: false
+              end
             end
           end
         end
diff --git a/lib/kamal/commands/app.rb b/lib/kamal/commands/app.rb
index cb1f7091..7fac2e17 100644
--- a/lib/kamal/commands/app.rb
+++ b/lib/kamal/commands/app.rb
@@ -1,34 +1,29 @@
 class Kamal::Commands::App < Kamal::Commands::Base
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 
-  attr_reader :role
+  attr_reader :role, :role_config
 
   def initialize(config, role: nil)
     super(config)
     @role = role
-  end
-
-  def start_or_run(hostname: nil)
-    combine start, run(hostname: hostname), by: "||"
+    @role_config = config.role(self.role)
   end
 
   def run(hostname: nil)
-    role = config.role(self.role)
-
     docker :run,
       "--detach",
       "--restart unless-stopped",
       "--name", container_name,
       *(["--hostname", hostname] if hostname),
       "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
-      *role.env_args,
-      *role.health_check_args,
+      *role_config.env_args,
+      *role_config.health_check_args,
       *config.logging_args,
       *config.volume_args,
-      *role.label_args,
-      *role.option_args,
+      *role_config.label_args,
+      *role_config.option_args,
       config.absolute_image,
-      role.cmd
+      role_config.cmd
   end
 
   def start
@@ -76,14 +71,12 @@ class Kamal::Commands::App < Kamal::Commands::Base
   end
 
   def execute_in_new_container(*command, interactive: false)
-    role = config.role(self.role)
-
     docker :run,
       ("-it" if interactive),
       "--rm",
-      *role&.env_args,
+      *role_config&.env_args,
       *config.volume_args,
-      *role&.option_args,
+      *role_config&.option_args,
       config.absolute_image,
       *command
   end
@@ -112,7 +105,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
   def list_versions(*docker_args, statuses: nil)
     pipe \
       docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
-      %(while read line; do echo ${line##{service_role_dest}-}; done) # Extract SHA from "service-role-dest-SHA"
+      %(while read line; do echo ${line##{role_config.full_name}-}; done) # Extract SHA from "service-role-dest-SHA"
   end
 
   def list_containers
@@ -150,16 +143,30 @@ class Kamal::Commands::App < Kamal::Commands::Base
   end
 
   def make_env_directory
-    make_directory config.role(role).host_env_directory
+    make_directory role_config.host_env_directory
   end
 
   def remove_env_file
-    [:rm, "-f", config.role(role).host_env_file_path]
+    [:rm, "-f", role_config.host_env_file_path]
+  end
+
+  def cord(version:)
+    pipe \
+      docker(:inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", container_name(version)),
+      [:awk, "'$2 == \"#{role_config.cord_container_directory}\" {print $1}'"]
+  end
+
+  def tie_cord(cord)
+    create_empty_file(cord)
+  end
+
+  def cut_cord(cord)
+    remove_directory(cord)
   end
 
   private
     def container_name(version = nil)
-      [ config.service, role, config.destination, version || config.version ].compact.join("-")
+      [ role_config.full_name, version || config.version ].compact.join("-")
     end
 
     def filter_args(statuses: nil)
diff --git a/lib/kamal/commands/base.rb b/lib/kamal/commands/base.rb
index 3058df16..ff31c747 100644
--- a/lib/kamal/commands/base.rb
+++ b/lib/kamal/commands/base.rb
@@ -34,6 +34,10 @@ module Kamal::Commands
       [ :mkdir, "-p", path ]
     end
 
+    def remove_directory(path)
+      [ :rm, "-r", path ]
+    end
+
     private
       def combine(*commands, by: "&&")
         commands
@@ -69,5 +73,11 @@ module Kamal::Commands
       def tags(**details)
         Kamal::Tags.from_config(config, **details)
       end
+
+      def create_empty_file(file)
+        chain \
+          make_directory_for(file),
+          [:touch, file]
+      end
   end
 end
diff --git a/lib/kamal/commands/healthcheck.rb b/lib/kamal/commands/healthcheck.rb
index 4327087d..fa050b9c 100644
--- a/lib/kamal/commands/healthcheck.rb
+++ b/lib/kamal/commands/healthcheck.rb
@@ -10,7 +10,7 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
       "--label", "service=#{container_name}",
       "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
       *web.env_args,
-      *web.health_check_args,
+      *web.health_check_args(cord: false),
       *config.volume_args,
       *web.option_args,
       config.absolute_image,
diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb
index 8db892fc..af6e46c5 100644
--- a/lib/kamal/configuration.rb
+++ b/lib/kamal/configuration.rb
@@ -61,6 +61,14 @@ class Kamal::Configuration
     raw_config.run_directory || ".kamal"
   end
 
+  def run_directory_as_docker_volume
+    if Pathname.new(run_directory).absolute?
+      run_directory
+    else
+      File.join "$(pwd)", run_directory
+    end
+  end
+
 
   def roles
     @roles ||= role_names.collect { |role_name| Role.new(role_name, config: self) }
@@ -141,7 +149,7 @@ class Kamal::Configuration
 
 
   def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999 }.merge(raw_config.healthcheck || {})
+    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }.merge(raw_config.healthcheck || {})
   end
 
   def readiness_delay
@@ -199,6 +207,10 @@ class Kamal::Configuration
     "#{run_directory}/env"
   end
 
+  def run_id
+    @run_id ||= SecureRandom.hex(16)
+  end
+
   private
     # Will raise ArgumentError if any required config keys are missing
     def ensure_required_keys_present
diff --git a/lib/kamal/configuration/role.rb b/lib/kamal/configuration/role.rb
index f549d459..6c57bc1d 100644
--- a/lib/kamal/configuration/role.rb
+++ b/lib/kamal/configuration/role.rb
@@ -1,4 +1,5 @@
 class Kamal::Configuration::Role
+  CORD_FILE = "cord"
   delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
 
   attr_accessor :name
@@ -47,28 +48,52 @@ class Kamal::Configuration::Role
     argumentize "--env-file", host_env_file_path
   end
 
-  def health_check_args
+  def health_check_args(cord: true)
     if health_check_cmd.present?
-      optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval })
+      if cord && uses_cord?
+        optionize({ "health-cmd" => health_check_cmd_with_cord, "health-interval" => health_check_interval })
+          .concat(["--volume", "#{cord_host_directory}:#{cord_container_directory}"])
+      else
+        optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval })
+      end
     else
       []
     end
   end
 
   def health_check_cmd
-    options = specializations["healthcheck"] || {}
-    options = config.healthcheck.merge(options) if running_traefik?
+    health_check_options["cmd"] || http_health_check(port: health_check_options["port"], path: health_check_options["path"])
+  end
 
-    options["cmd"] || http_health_check(port: options["port"], path: options["path"])
+  def health_check_cmd_with_cord
+    "(#{health_check_cmd}) && (stat #{cord_container_file} > /dev/null || exit 1)"
   end
 
   def health_check_interval
-    options = specializations["healthcheck"] || {}
-    options = config.healthcheck.merge(options) if running_traefik?
-
-    options["interval"] || "1s"
+    health_check_options["interval"] || "1s"
   end
 
+  def uses_cord?
+    running_traefik? && cord_container_directory.present? && health_check_cmd.present?
+  end
+
+  def cord_host_directory
+    File.join config.run_directory_as_docker_volume, "cords", [full_name, config.run_id].join("-")
+  end
+
+  def cord_host_file
+    File.join cord_host_directory, CORD_FILE
+  end
+
+  def cord_container_directory
+    health_check_options.fetch("cord", nil)
+  end
+
+  def cord_container_file
+    File.join cord_container_directory, CORD_FILE
+  end
+
+
   def cmd
     specializations["cmd"]
   end
@@ -85,6 +110,10 @@ class Kamal::Configuration::Role
     name.web? || specializations["traefik"]
   end
 
+  def full_name
+    [ config.service, name, config.destination ].compact.join("-")
+  end
+
   private
     attr_accessor :config
 
@@ -164,4 +193,12 @@ class Kamal::Configuration::Role
     def http_health_check(port:, path:)
       "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present?
     end
+
+    def health_check_options
+      @health_check_options ||= begin
+        options = specializations["healthcheck"] || {}
+        options = config.healthcheck.merge(options) if running_traefik?
+        options
+      end
+    end
 end
diff --git a/lib/kamal/utils/healthcheck_poller.rb b/lib/kamal/utils/healthcheck_poller.rb
index ddb09ec6..27a2ff7d 100644
--- a/lib/kamal/utils/healthcheck_poller.rb
+++ b/lib/kamal/utils/healthcheck_poller.rb
@@ -1,5 +1,5 @@
 class Kamal::Utils::HealthcheckPoller
-  TRAEFIK_HEALTHY_DELAY = 2
+  TRAEFIK_UPDATE_DELAY = 2
 
   class HealthcheckError < StandardError; end
 
@@ -11,7 +11,7 @@ class Kamal::Utils::HealthcheckPoller
       begin
         case status = block.call
         when "healthy"
-          sleep TRAEFIK_HEALTHY_DELAY if pause_after_ready
+          sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
         when "running" # No health check configured
           sleep KAMAL.config.readiness_delay if pause_after_ready
         else
@@ -31,6 +31,31 @@ class Kamal::Utils::HealthcheckPoller
       info "Container is healthy!"
     end
 
+    def wait_for_unhealthy(pause_after_ready: false, &block)
+      attempt = 1
+      max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+      begin
+        case status = block.call
+        when "unhealthy"
+          sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
+        else
+          raise HealthcheckError, "container not unhealthy (#{status})"
+        end
+      rescue HealthcheckError => e
+        if attempt <= max_attempts
+          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+          sleep attempt
+          attempt += 1
+          retry
+        else
+          raise
+        end
+      end
+
+      info "Container is unhealthy!"
+    end
+
     private
       def info(message)
         SSHKit.config.output.info(message)
diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb
index 8a90e90a..d0e6268a 100644
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -11,10 +11,11 @@ class CliAppTest < CliTestCase
   end
 
   test "boot will rename if same version is already running" do
+    Object.any_instance.stubs(:sleep)
     run_command("details") # Preheat Kamal const
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
       .returns("12345678") # running version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -25,6 +26,14 @@ class CliAppTest < CliTestCase
       .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
       .returns("123") # old version
 
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .returns("cordfile") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("unhealthy") # old version unhealthy
+
     run_command("boot").tap do |output|
       assert_match /Renaming container .* to .* as already deployed on 1.1.1.1/, output # Rename
       assert_match /docker rename app-web-latest app-web-latest_replaced_[0-9a-f]{16}/, output
@@ -180,10 +189,16 @@ class CliAppTest < CliTestCase
     end
 
     def stub_running
+      Object.any_instance.stubs(:sleep)
+
       SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
 
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
         .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
         .returns("running") # health check
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+        .returns("unhealthy") # health check
     end
 end
diff --git a/test/cli/healthcheck_test.rb b/test/cli/healthcheck_test.rb
index f9c3aa9c..b07c4900 100644
--- a/test/cli/healthcheck_test.rb
+++ b/test/cli/healthcheck_test.rb
@@ -6,6 +6,7 @@ class CliHealthcheckTest < CliTestCase
     Thread.report_on_exception = false
 
     Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index 04f11575..28ef97ff 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -176,9 +176,10 @@ class CliMainTest < CliTestCase
   end
 
   test "rollback good version" do
+    Object.any_instance.stubs(:sleep)
     [ "web", "workers" ].each do |role|
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :container, :ls, "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false)
+        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false)
         .returns("").at_least_once
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
         .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet")
@@ -191,14 +192,21 @@ class CliMainTest < CliTestCase
         .returns("running").at_least_once # health check
     end
 
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .returns("corddirectory").at_least_once # health check
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-version-to-rollback$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("unhealthy").at_least_once # health check
+
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     hook_variables = { version: 123, service_version: "app@123", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "rollback" }
 
     run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
-      assert_match "Start container with version 123", output
       assert_hook_ran "pre-deploy", output, **hook_variables
       assert_match "docker tag dhh/app:123 dhh/app:latest", output
-      assert_match "docker start app-web-123", output
+      assert_match "docker run --detach --restart unless-stopped --name app-web-123", output
       assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running"
       assert_hook_ran "post-deploy", output, **hook_variables, runtime: "0"
     end
@@ -210,7 +218,7 @@ class CliMainTest < CliTestCase
     Kamal::Utils::HealthcheckPoller.stubs(:sleep)
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
       .returns("").at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
@@ -220,8 +228,7 @@ class CliMainTest < CliTestCase
       .returns("running").at_least_once # health check
 
     run_command("rollback", "123").tap do |output|
-      assert_match "Start container with version 123", output
-      assert_match "docker start app-web-123 || docker run --detach --restart unless-stopped --name app-web-123", output
+      assert_match "docker run --detach --restart unless-stopped --name app-web-123", output
       assert_no_match "docker stop", output
     end
   end
diff --git a/test/cli/traefik_test.rb b/test/cli/traefik_test.rb
index 1b19b033..6c6fbf64 100644
--- a/test/cli/traefik_test.rb
+++ b/test/cli/traefik_test.rb
@@ -19,6 +19,8 @@ class CliTraefikTest < CliTestCase
   end
 
   test "reboot --rolling" do
+    Object.any_instance.stubs(:sleep)
+
     run_command("reboot", "--rolling").tap do |output|
       assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.1", output
     end
diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb
index ab3f6ace..7b769c6d 100644
--- a/test/commands/app_test.rb
+++ b/test/commands/app_test.rb
@@ -3,6 +3,7 @@ require "test_helper"
 class CommandsAppTest < ActiveSupport::TestCase
   setup do
     ENV["RAILS_MASTER_KEY"] = "456"
+    Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
 
     @config = { service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], env: { "secret" => [ "RAILS_MASTER_KEY" ] } }
   end
@@ -13,13 +14,13 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with hostname" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run(hostname: "myhost").join(" ")
   end
 
@@ -27,7 +28,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:volumes] = ["/local/path:/container/path" ]
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -35,7 +36,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "path" => "/healthz" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/healthz || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -43,7 +44,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "cmd" => "/bin/up" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/up\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -51,7 +52,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"/bin/healthy\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -66,7 +67,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -83,18 +84,6 @@ class CommandsAppTest < ActiveSupport::TestCase
       new_command.start.join(" ")
   end
 
-  test "start_or_run" do
-    assert_equal \
-      "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
-      new_command.start_or_run.join(" ")
-  end
-
-  test "start_or_run with hostname" do
-    assert_equal \
-      "docker start app-web-999 || docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
-      new_command.start_or_run(hostname: "myhost").join(" ")
-  end
-
   test "stop" do
     assert_equal \
       "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop",
@@ -342,6 +331,20 @@ class CommandsAppTest < ActiveSupport::TestCase
     assert_equal "rm -f .kamal/env/roles/app-web.env", new_command.remove_env_file.join(" ")
   end
 
+  test "cord" do
+    assert_equal "docker inspect -f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}' app-web-123 | awk '$2 == \"/tmp/kamal-cord\" {print $1}'", new_command.cord(version: 123).join(" ")
+  end
+
+  test "tie cord" do
+    assert_equal "mkdir -p . ; touch cordfile", new_command.tie_cord("cordfile").join(" ")
+    assert_equal "mkdir -p corddir ; touch corddir/cordfile", new_command.tie_cord("corddir/cordfile").join(" ")
+    assert_equal "mkdir -p /corddir ; touch /corddir/cordfile", new_command.tie_cord("/corddir/cordfile").join(" ")
+  end
+
+  test "cut cord" do
+    assert_equal "rm -r corddir", new_command.cut_cord("corddir").join(" ")
+  end
+
   private
     def new_command(role: "web")
       Kamal::Commands::App.new(Kamal::Configuration.new(@config, destination: @destination, version: "999"), role: role)
diff --git a/test/configuration/role_test.rb b/test/configuration/role_test.rb
index b29ac2b5..65003979 100644
--- a/test/configuration/role_test.rb
+++ b/test/configuration/role_test.rb
@@ -175,4 +175,24 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).host_env_file_path
   end
 
+  test "uses cord" do
+    assert @config_with_roles.role(:web).uses_cord?
+    assert !@config_with_roles.role(:workers).uses_cord?
+  end
+
+  test "cord host directory" do
+    assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}}, @config_with_roles.role(:web).cord_host_directory
+  end
+
+  test "cord host file" do
+    assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}/cord}, @config_with_roles.role(:web).cord_host_file
+  end
+
+  test "cord container directory" do
+    assert_equal "/tmp/kamal-cord", @config_with_roles.role(:web).cord_container_directory
+  end
+
+  test "cord container file" do
+    assert_equal "/tmp/kamal-cord/cord", @config_with_roles.role(:web).cord_container_file
+  end
 end
diff --git a/test/configuration_test.rb b/test/configuration_test.rb
index d31e0dbd..59046760 100644
--- a/test/configuration_test.rb
+++ b/test/configuration_test.rb
@@ -224,7 +224,7 @@ class ConfigurationTest < ActiveSupport::TestCase
         :volume_args=>["--volume", "/local/path:/container/path"],
         :builder=>{},
         :logging=>["--log-opt", "max-size=\"10m\""],
-        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999 }}
+        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }}
 
     assert_equal expected_config, @config.to_h
   end
@@ -252,4 +252,17 @@ class ConfigurationTest < ActiveSupport::TestCase
     config = Kamal::Configuration.new(@deploy.merge!(run_directory: "/root/kamal"))
     assert_equal "/root/kamal", config.run_directory
   end
+
+  test "run directory as docker volume" do
+    config = Kamal::Configuration.new(@deploy)
+    assert_equal "$(pwd)/.kamal", config.run_directory_as_docker_volume
+
+    config = Kamal::Configuration.new(@deploy.merge!(run_directory: "/root/kamal"))
+    assert_equal "/root/kamal", config.run_directory_as_docker_volume
+  end
+
+  test "run id" do
+    SecureRandom.expects(:hex).with(16).returns("09876543211234567890098765432112")
+    assert_equal "09876543211234567890098765432112", @config.run_id
+  end
 end
diff --git a/test/integration/docker/deployer/Dockerfile b/test/integration/docker/deployer/Dockerfile
index b964e0e3..2b49fcb0 100644
--- a/test/integration/docker/deployer/Dockerfile
+++ b/test/integration/docker/deployer/Dockerfile
@@ -2,6 +2,8 @@ FROM ruby:3.2
 
 WORKDIR /app
 
+ENV VERBOSE=true
+
 RUN apt-get update --fix-missing && apt-get install -y ca-certificates openssh-client curl gnupg docker.io
 
 RUN install -m 0755 -d /etc/apt/keyrings
diff --git a/test/integration/docker/deployer/app/config/deploy.yml b/test/integration/docker/deployer/app/config/deploy.yml
index 6ecb94b3..fe4a26ff 100644
--- a/test/integration/docker/deployer/app/config/deploy.yml
+++ b/test/integration/docker/deployer/app/config/deploy.yml
@@ -18,7 +18,7 @@ builder:
   args:
     COMMIT_SHA: <%= `git rev-parse HEAD` %>
 healthcheck:
-  cmd: wget -qO- http://localhost > /dev/null
+  cmd: wget -qO- http://localhost > /dev/null || exit 1
 traefik:
   args:
     accesslog: true
diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb
index 08e2567f..e10d9e52 100644
--- a/test/integration/main_test.rb
+++ b/test/integration/main_test.rb
@@ -54,7 +54,7 @@ class MainTest < IntegrationTest
     assert_equal({ user: "root", auth_methods: [ "publickey" ], keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])
     assert_equal({ "multiarch" => false, "args" => { "COMMIT_SHA" => version } }, config[:builder])
     assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging]
-    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cmd" => "wget -qO- http://localhost > /dev/null" }, config[:healthcheck])
+    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord"=>"/tmp/kamal-cord", "cmd"=>"wget -qO- http://localhost > /dev/null || exit 1" }, config[:healthcheck])
   end
 
   private