Bump version for 2.6.1

Per role proxy docs

Dockerfile

@@ -15,7 +15,7 @@ COPY lib/kamal/version.rb /kamal/lib/kamal/version.rb
 # Install system dependencies
 RUN apk add --no-cache build-base git docker openrc openssh-client-default yaml-dev \
     && rc-update add docker boot \
-    && gem install bundler --version=2.4.3 \
+    && gem install bundler --version=2.6.5 \
     && bundle install
 
 # Copy the rest of our application code into the container.

Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    kamal (2.5.2)
+    kamal (2.6.1)
       activesupport (>= 7.0)
       base64 (~> 0.2)
       bcrypt_pbkdf (~> 1.0)
       concurrent-ruby (~> 1.2)
       dotenv (~> 3.1)
-      ed25519 (~> 1.2)
+      ed25519 (~> 1.4)
       net-ssh (~> 7.3)
       sshkit (>= 1.23.0, < 2.0)
       thor (~> 1.3)
@@ -48,8 +48,6 @@ GEM
     ast (2.4.2)
     base64 (0.2.0)
     bcrypt_pbkdf (1.1.1)
-    bcrypt_pbkdf (1.1.1-arm64-darwin)
-    bcrypt_pbkdf (1.1.1-x86_64-darwin)
     benchmark (0.4.0)
     bigdecimal (3.1.8)
     builder (3.3.0)
@@ -62,7 +60,7 @@ GEM
       reline (>= 0.3.8)
     dotenv (3.1.5)
     drb (2.2.1)
-    ed25519 (1.3.0)
+    ed25519 (1.4.0)
     erubi (1.13.0)
     i18n (1.14.6)
       concurrent-ruby (~> 1.0)
@@ -84,11 +82,15 @@ GEM
     net-sftp (4.0.0)
       net-ssh (>= 5.0.0, < 8.0.0)
     net-ssh (7.3.0)
-    nokogiri (1.17.2-arm64-darwin)
+    nokogiri (1.18.8-aarch64-linux-musl)
       racc (~> 1.4)
-    nokogiri (1.17.2-x86_64-darwin)
+    nokogiri (1.18.8-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.17.2-x86_64-linux)
+    nokogiri (1.18.8-x86_64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.18.8-x86_64-linux-gnu)
+      racc (~> 1.4)
+    nokogiri (1.18.8-x86_64-linux-musl)
       racc (~> 1.4)
     ostruct (0.6.1)
     parallel (1.26.3)
@@ -99,7 +101,7 @@ GEM
       date
       stringio
     racc (1.8.1)
-    rack (3.1.8)
+    rack (3.1.12)
     rack-session (2.0.0)
       rack (>= 3.0.0)
     rack-test (2.1.0)
@@ -172,14 +174,16 @@ GEM
     unicode-display_width (3.1.2)
       unicode-emoji (~> 4.0, >= 4.0.4)
     unicode-emoji (4.0.4)
-    uri (1.0.2)
+    uri (1.0.3)
     useragent (0.16.11)
     zeitwerk (2.7.1)
 
 PLATFORMS
+  aarch64-linux-musl
   arm64-darwin
   x86_64-darwin
   x86_64-linux
+  x86_64-linux-musl
 
 DEPENDENCIES
   debug
@@ -189,4 +193,4 @@ DEPENDENCIES
   rubocop-rails-omakase
 
 BUNDLED WITH
-   2.4.3
+   2.6.5

kamal.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "thor", "~> 1.3"
   spec.add_dependency "dotenv", "~> 3.1"
   spec.add_dependency "zeitwerk", ">= 2.6.18", "< 3.0"
-  spec.add_dependency "ed25519", "~> 1.2"
+  spec.add_dependency "ed25519", "~> 1.4"
   spec.add_dependency "bcrypt_pbkdf", "~> 1.0"
   spec.add_dependency "concurrent-ruby", "~> 1.2"
   spec.add_dependency "base64", "~> 0.2"

lib/kamal/cli/accessory.rb

@@ -1,4 +1,5 @@
 require "active_support/core_ext/array/conversions"
+require "concurrent/array"
 
 class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "boot [NAME]", "Boot new accessory service on host (use NAME=all to boot all accessories)"
@@ -10,6 +11,16 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
         prepare(name) if prepare
 
         with_accessory(name) do |accessory, hosts|
+          booted_hosts = Concurrent::Array.new
+          on(hosts) do |host|
+            booted_hosts << host.to_s if capture_with_info(*accessory.info(all: true, quiet: true)).strip.presence
+          end
+
+          if booted_hosts.any?
+            say "Skipping booting `#{name}` on #{booted_hosts.sort.join(", ")}, a container already exists", :yellow
+            hosts -= booted_hosts
+          end
+
           directories(name)
           upload(name)
 
@@ -130,6 +141,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
   option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
   def exec(name, *cmd)
+    pre_connect_if_required
+
     cmd = Kamal::Utils.join_commands(cmd)
     with_accessory(name) do |accessory, hosts|
       case
@@ -139,6 +152,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
       when options[:interactive]
         say "Launching interactive command via SSH from new container...", :magenta
+        on(accessory.hosts.first) { execute *KAMAL.registry.login }
         run_locally { exec accessory.execute_in_new_container_over_ssh(cmd) }
 
       when options[:reuse]
@@ -151,6 +165,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
       else
         say "Launching command from new container...", :magenta
         on(hosts) do |host|
+          execute *KAMAL.registry.login
           execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
           puts_by_host host, capture_with_info(*accessory.execute_in_new_container(cmd))
         end
@@ -275,11 +290,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     end
 
     def accessory_hosts(accessory)
-      if KAMAL.specific_hosts&.any?
+      KAMAL.accessory_hosts & accessory.hosts
-        KAMAL.specific_hosts & accessory.hosts
-      else
-        accessory.hosts
-      end
     end
 
     def remove_accessory(name)

lib/kamal/cli/app.rb

@@ -7,9 +7,11 @@ class Kamal::Cli::App < Kamal::Cli::Base
         say "Start container with version #{version} (or reboot if already running)...", :magenta
 
         # Assets are prepared in a separate step to ensure they are on all hosts before booting
-        on(KAMAL.hosts) do
+        on(KAMAL.app_hosts) do
+          Kamal::Cli::App::ErrorPages.new(host, self).run
+
           KAMAL.roles_on(host).each do |role|
-            Kamal::Cli::App::PrepareAssets.new(host, role, self).run
+            Kamal::Cli::App::Assets.new(host, role, self).run
           end
         end
 
@@ -31,7 +33,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
         end
 
         # Tag once the app booted on all hosts
-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
           execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
           execute *KAMAL.app.tag_latest_image
         end
@@ -42,7 +44,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "start", "Start existing app container on servers"
   def start
     with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
@@ -65,7 +67,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "stop", "Stop app container on servers"
   def stop
     with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
@@ -89,7 +91,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
   # FIXME: Drop in favor of just containers?
   desc "details", "Show details about app containers"
   def details
-    on(KAMAL.hosts) do |host|
+    on(KAMAL.app_hosts) do |host|
       roles = KAMAL.roles_on(host)
 
       roles.each do |role|
@@ -104,10 +106,16 @@ class Kamal::Cli::App < Kamal::Cli::Base
   option :env, aliases: "-e", type: :hash, desc: "Set environment variables for the command"
   option :detach, type: :boolean, default: false, desc: "Execute command in a detached container"
   def exec(*cmd)
+    pre_connect_if_required
+
     if (incompatible_options = [ :interactive, :reuse ].select { |key| options[:detach] && options[key] }.presence)
       raise ArgumentError, "Detach is not compatible with #{incompatible_options.join(" or ")}"
     end
 
+    if cmd.empty?
+      raise ArgumentError, "No command provided. You must specify a command to execute."
+    end
+
     cmd = Kamal::Utils.join_commands(cmd)
     env = options[:env]
     detach = options[:detach]
@@ -123,6 +131,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
       say "Get most recent version available as an image...", :magenta unless options[:version]
       using_version(version_or_latest) do |version|
         say "Launching interactive command with version #{version} via SSH from new container on #{KAMAL.primary_host}...", :magenta
+        on(KAMAL.primary_host) { execute *KAMAL.registry.login }
         run_locally do
           exec KAMAL.app(role: KAMAL.primary_role, host: KAMAL.primary_host).execute_in_new_container_over_ssh(cmd, env: env)
         end
@@ -133,7 +142,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
       using_version(options[:version] || current_running_version) do |version|
         say "Launching command with version #{version} from existing container...", :magenta
 
-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
           roles = KAMAL.roles_on(host)
 
           roles.each do |role|
@@ -147,7 +156,9 @@ class Kamal::Cli::App < Kamal::Cli::Base
       say "Get most recent version available as an image...", :magenta unless options[:version]
       using_version(version_or_latest) do |version|
         say "Launching command with version #{version} from new container...", :magenta
-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
+          execute *KAMAL.registry.login
+
           roles = KAMAL.roles_on(host)
 
           roles.each do |role|
@@ -161,7 +172,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "containers", "Show app containers on servers"
   def containers
-    on(KAMAL.hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_containers) }
+    on(KAMAL.app_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_containers) }
   end
 
   desc "stale_containers", "Detect app stale containers"
@@ -170,7 +181,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
     stop = options[:stop]
 
     with_lock_if_stopping do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
@@ -193,7 +204,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "images", "Show app images on servers"
   def images
-    on(KAMAL.hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_images) }
+    on(KAMAL.app_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_images) }
   end
 
   desc "logs", "Show log lines from app on servers (use --help to show options)"
@@ -229,7 +240,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
     else
       lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
@@ -249,14 +260,44 @@ class Kamal::Cli::App < Kamal::Cli::Base
       stop
       remove_containers
       remove_images
-      remove_app_directory
+      remove_app_directories
+    end
+  end
+
+  desc "live", "Set the app to live mode"
+  def live
+    with_lock do
+      on(KAMAL.proxy_hosts) do |host|
+        roles = KAMAL.roles_on(host)
+
+        roles.each do |role|
+          execute *KAMAL.app(role: role, host: host).live if role.running_proxy?
+        end
+      end
+    end
+  end
+
+  desc "maintenance", "Set the app to maintenance mode"
+  option :drain_timeout, type: :numeric, desc: "How long to allow in-flight requests to complete (defaults to drain_timeout from config)"
+  option :message, type: :string, desc: "Message to display to clients while stopped"
+  def maintenance
+    maintenance_options = { drain_timeout: options[:drain_timeout] || KAMAL.config.drain_timeout, message: options[:message] }
+
+    with_lock do
+      on(KAMAL.proxy_hosts) do |host|
+        roles = KAMAL.roles_on(host)
+
+        roles.each do |role|
+          execute *KAMAL.app(role: role, host: host).maintenance(**maintenance_options) if role.running_proxy?
+        end
+      end
     end
   end
 
   desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true
   def remove_container(version)
     with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
@@ -270,7 +311,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "remove_containers", "Remove all app containers from servers", hide: true
   def remove_containers
     with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
@@ -284,30 +325,33 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "remove_images", "Remove all app images from servers", hide: true
   def remove_images
     with_lock do
-      on(KAMAL.hosts) do
+      on(KAMAL.app_hosts) do
         execute *KAMAL.auditor.record("Removed all app images"), verbosity: :debug
         execute *KAMAL.app.remove_images
       end
     end
   end
 
-  desc "remove_app_directory", "Remove the service directory from servers", hide: true
+  desc "remove_app_directories", "Remove the app directories from servers", hide: true
-  def remove_app_directory
+  def remove_app_directories
     with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
-          execute *KAMAL.auditor.record("Removed #{KAMAL.config.app_directory} on all servers", role: role), verbosity: :debug
+          execute *KAMAL.auditor.record("Removed #{KAMAL.config.app_directory}", role: role), verbosity: :debug
           execute *KAMAL.server.remove_app_directory, raise_on_non_zero_exit: false
         end
+
+        execute *KAMAL.auditor.record("Removed #{KAMAL.config.app_directory}"), verbosity: :debug
+        execute *KAMAL.app.remove_proxy_app_directory, raise_on_non_zero_exit: false
       end
     end
   end
 
   desc "version", "Show app version currently running on servers"
   def version
-    on(KAMAL.hosts) do |host|
+    on(KAMAL.app_hosts) do |host|
       role = KAMAL.roles_on(host).first
       puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).current_running_version).strip
     end
@@ -350,6 +394,6 @@ class Kamal::Cli::App < Kamal::Cli::Base
     end
 
     def host_boot_groups
-      KAMAL.config.boot.limit ? KAMAL.hosts.each_slice(KAMAL.config.boot.limit).to_a : [ KAMAL.hosts ]
+      KAMAL.config.boot.limit ? KAMAL.app_hosts.each_slice(KAMAL.config.boot.limit).to_a : [ KAMAL.app_hosts ]
     end
 end

lib/kamal/cli/app/assets.rb

@@ -1,4 +1,4 @@
-class Kamal::Cli::App::PrepareAssets
+class Kamal::Cli::App::Assets
   attr_reader :host, :role, :sshkit
   delegate :execute, :capture_with_info, :info, to: :sshkit
   delegate :assets?, to: :role

lib/kamal/cli/app/boot.rb

@@ -70,6 +70,7 @@ class Kamal::Cli::App::Boot
     def stop_old_version(version)
       execute *app.stop(version: version), raise_on_non_zero_exit: false
       execute *app.clean_up_assets if assets?
+      execute *app.clean_up_error_pages if KAMAL.config.error_pages_path
     end
 
     def release_barrier

lib/kamal/cli/app/error_pages.rb

@@ -0,0 +1,33 @@
+class Kamal::Cli::App::ErrorPages
+  ERROR_PAGES_GLOB = "{4??.html,5??.html}"
+
+  attr_reader :host, :sshkit
+  delegate :upload!, :execute, to: :sshkit
+
+  def initialize(host, sshkit)
+    @host = host
+    @sshkit = sshkit
+  end
+
+  def run
+    if KAMAL.config.error_pages_path
+      with_error_pages_tmpdir do |local_error_pages_dir|
+        execute *KAMAL.app.create_error_pages_directory
+        upload! local_error_pages_dir, KAMAL.config.proxy_boot.error_pages_directory, mode: "0700", recursive: true
+      end
+    end
+  end
+
+  private
+    def with_error_pages_tmpdir
+      Dir.mktmpdir("kamal-error-pages") do |tmpdir|
+        error_pages_dir = File.join(tmpdir, KAMAL.config.version)
+        FileUtils.mkdir(error_pages_dir)
+
+        if (files = Dir[File.join(KAMAL.config.error_pages_path, ERROR_PAGES_GLOB)]).any?
+          FileUtils.cp(files, error_pages_dir)
+          yield error_pages_dir
+        end
+      end
+    end
+end

lib/kamal/cli/base.rb

@@ -133,7 +133,13 @@ module Kamal::Cli
 
       def run_hook(hook, **extra_details)
         if !options[:skip_hooks] && KAMAL.hook.hook_exists?(hook)
-          details = { hosts: KAMAL.hosts.join(","), command: command, subcommand: subcommand }
+          details = {
+            hosts: KAMAL.hosts.join(","),
+            roles: KAMAL.specific_roles&.join(","),
+            lock: KAMAL.holding_lock?.to_s,
+            command: command,
+            subcommand: subcommand
+          }.compact
 
           say "Running the #{hook} hook...", :magenta
           with_env KAMAL.hook.env(**details, **extra_details) do
@@ -147,12 +153,16 @@ module Kamal::Cli
       end
 
       def on(*args, &block)
+        pre_connect_if_required
+
+        super
+      end
+
+      def pre_connect_if_required
         if !KAMAL.connected?
           run_hook "pre-connect"
           KAMAL.connected = true
         end
-
-        super
       end
 
       def command

lib/kamal/cli/build.rb

@@ -14,7 +14,13 @@ class Kamal::Cli::Build < Kamal::Cli::Base
   def push
     cli = self
 
+    # Ensure pre-connect hooks run before the build, they may needed for a remote builder
+    # or the pre-build hooks.
+    pre_connect_if_required
+
     ensure_docker_installed
+    login_to_registry_locally
+
     run_hook "pre-build"
 
     uncommitted_changes = Kamal::Git.uncommitted_changes
@@ -61,14 +67,16 @@ class Kamal::Cli::Build < Kamal::Cli::Base
 
   desc "pull", "Pull app image from registry onto servers"
   def pull
+    login_to_registry_remotely
+
     if (first_hosts = mirror_hosts).any?
       #  Pull on a single host per mirror first to seed them
       say "Pulling image on #{first_hosts.join(", ")} to seed the #{"mirror".pluralize(first_hosts.count)}...", :magenta
       pull_on_hosts(first_hosts)
       say "Pulling image on remaining hosts...", :magenta
-      pull_on_hosts(KAMAL.hosts - first_hosts)
+      pull_on_hosts(KAMAL.app_hosts - first_hosts)
     else
-      pull_on_hosts(KAMAL.hosts)
+      pull_on_hosts(KAMAL.app_hosts)
     end
   end
 
@@ -159,9 +167,9 @@ class Kamal::Cli::Build < Kamal::Cli::Base
     end
 
     def mirror_hosts
-      if KAMAL.hosts.many?
+      if KAMAL.app_hosts.many?
         mirror_hosts = Concurrent::Hash.new
-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
           first_mirror = capture_with_info(*KAMAL.builder.first_mirror).strip.presence
           mirror_hosts[first_mirror] ||= host.to_s if first_mirror
         rescue SSHKit::Command::Failed => e
@@ -181,4 +189,16 @@ class Kamal::Cli::Build < Kamal::Cli::Base
         execute *KAMAL.builder.validate_image
       end
     end
+
+    def login_to_registry_locally
+      run_locally do
+        execute *KAMAL.registry.login
+      end
+    end
+
+    def login_to_registry_remotely
+      on(KAMAL.app_hosts) do
+        execute *KAMAL.registry.login
+      end
+    end
 end

lib/kamal/cli/main.rb

@@ -20,9 +20,6 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     runtime = print_runtime do
       invoke_options = deploy_options
 
-      say "Log into image registry...", :magenta
-      invoke "kamal:cli:registry:login", [], invoke_options.merge(skip_local: options[:skip_push])
-
       if options[:skip_push]
         say "Pull app image...", :magenta
         invoke "kamal:cli:build:pull", [], invoke_options
@@ -52,7 +49,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s
   end
 
-  desc "redeploy", "Deploy app to servers without bootstrapping servers, starting kamal-proxy, pruning, and registry login"
+  desc "redeploy", "Deploy app to servers without bootstrapping servers, starting kamal-proxy and pruning"
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def redeploy
     runtime = print_runtime do
@@ -197,10 +194,10 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     confirming "This will replace Traefik with kamal-proxy and restart all accessories" do
       with_lock do
         if options[:rolling]
-          (KAMAL.hosts | KAMAL.accessory_hosts).each do |host|
+          KAMAL.hosts.each do |host|
             KAMAL.with_specific_hosts(host) do
               say "Upgrading #{host}...", :magenta
-              if KAMAL.hosts.include?(host)
+              if KAMAL.app_hosts.include?(host)
                 invoke "kamal:cli:proxy:upgrade", [], options.merge(confirmed: true, rolling: false)
                 reset_invocation(Kamal::Cli::Proxy)
               end
@@ -256,7 +253,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   private
     def container_available?(version)
       begin
-        on(KAMAL.hosts) do
+        on(KAMAL.app_hosts) do
           KAMAL.roles_on(host).each do |role|
             container_id = capture_with_info(*KAMAL.app(role: role, host: host).container_id_for_version(version))
             raise "Container not found" unless container_id.present?

lib/kamal/cli/proxy.rb

@@ -13,9 +13,10 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
 
         version = capture_with_info(*KAMAL.proxy.version).strip.presence
 
-        if version && Kamal::Utils.older_version?(version, Kamal::Configuration::PROXY_MINIMUM_VERSION)
+        if version && Kamal::Utils.older_version?(version, Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION)
-          raise "kamal-proxy version #{version} is too old, run `kamal proxy reboot` in order to update to at least #{Kamal::Configuration::PROXY_MINIMUM_VERSION}"
+          raise "kamal-proxy version #{version} is too old, run `kamal proxy reboot` in order to update to at least #{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}"
         end
+        execute *KAMAL.proxy.ensure_apps_config_directory
         execute *KAMAL.proxy.start_or_run
       end
     end
@@ -24,30 +25,75 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
   desc "boot_config <set|get|reset>", "Manage kamal-proxy boot configuration"
   option :publish, type: :boolean, default: true, desc: "Publish the proxy ports on the host"
   option :publish_host_ip, type: :string, repeatable: true, default: nil, desc: "Host IP address to bind HTTP/HTTPS traffic to. Defaults to all interfaces"
-  option :http_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTP_PORT, desc: "HTTP port to publish on the host"
+  option :http_port, type: :numeric, default: Kamal::Configuration::Proxy::Boot::DEFAULT_HTTP_PORT, desc: "HTTP port to publish on the host"
-  option :https_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTPS_PORT, desc: "HTTPS port to publish on the host"
+  option :https_port, type: :numeric, default: Kamal::Configuration::Proxy::Boot::DEFAULT_HTTPS_PORT, desc: "HTTPS port to publish on the host"
-  option :log_max_size, type: :string, default: Kamal::Configuration::PROXY_LOG_MAX_SIZE, desc: "Max size of proxy logs"
+  option :log_max_size, type: :string, default: Kamal::Configuration::Proxy::Boot::DEFAULT_LOG_MAX_SIZE, desc: "Max size of proxy logs"
+  option :registry, type: :string, default: nil, desc: "Registry to use for the proxy image"
+  option :repository, type: :string, default: nil, desc: "Repository for the proxy image"
+  option :image_version, type: :string, default: nil, desc: "Version of the proxy to run"
+  option :metrics_port, type: :numeric, default: nil, desc: "Port to report prometheus metrics on"
+  option :debug, type: :boolean, default: false, desc: "Whether to run the proxy in debug mode"
   option :docker_options, type: :array, default: [], desc: "Docker options to pass to the proxy container", banner: "option=value option2=value2"
   def boot_config(subcommand)
+    proxy_boot_config = KAMAL.config.proxy_boot
+
     case subcommand
     when "set"
       boot_options = [
-        *(KAMAL.config.proxy_publish_args(options[:http_port], options[:https_port], options[:publish_host_ip]) if options[:publish]),
+        *(proxy_boot_config.publish_args(options[:http_port], options[:https_port], options[:publish_host_ip]) if options[:publish]),
-        *(KAMAL.config.proxy_logging_args(options[:log_max_size])),
+        *(proxy_boot_config.logging_args(options[:log_max_size])),
+        *("--expose=#{options[:metrics_port]}" if options[:metrics_port]),
         *options[:docker_options].map { |option| "--#{option}" }
       ]
 
+      image = [
+        options[:registry].presence,
+        options[:repository].presence || proxy_boot_config.repository_name,
+        proxy_boot_config.image_name
+      ].compact.join("/")
+
+      image_version = options[:image_version]
+
+      run_command_options = { debug: options[:debug] || nil, "metrics-port": options[:metrics_port] }.compact
+      run_command = "kamal-proxy run #{Kamal::Utils.optionize(run_command_options).join(" ")}" if run_command_options.any?
+
       on(KAMAL.proxy_hosts) do |host|
         execute(*KAMAL.proxy.ensure_proxy_directory)
-        upload! StringIO.new(boot_options.join(" ")), KAMAL.config.proxy_options_file
+        if boot_options != proxy_boot_config.default_boot_options
+          upload! StringIO.new(boot_options.join(" ")), proxy_boot_config.options_file
+        else
+          execute *KAMAL.proxy.reset_boot_options, raise_on_non_zero_exit: false
+        end
+
+        if image != proxy_boot_config.image_default
+          upload! StringIO.new(image), proxy_boot_config.image_file
+        else
+          execute *KAMAL.proxy.reset_image, raise_on_non_zero_exit: false
+        end
+
+        if image_version
+          upload! StringIO.new(image_version), proxy_boot_config.image_version_file
+        else
+          execute *KAMAL.proxy.reset_image_version, raise_on_non_zero_exit: false
+        end
+
+        if run_command
+          upload! StringIO.new(run_command), proxy_boot_config.run_command_file
+        else
+          execute *KAMAL.proxy.reset_run_command, raise_on_non_zero_exit: false
+        end
       end
     when "get"
+
       on(KAMAL.proxy_hosts) do |host|
-        puts "Host #{host}: #{capture_with_info(*KAMAL.proxy.get_boot_options)}"
+        puts "Host #{host}: #{capture_with_info(*KAMAL.proxy.boot_config)}"
       end
     when "reset"
       on(KAMAL.proxy_hosts) do |host|
-        execute *KAMAL.proxy.reset_boot_options
+        execute *KAMAL.proxy.reset_boot_options, raise_on_non_zero_exit: false
+        execute *KAMAL.proxy.reset_image, raise_on_non_zero_exit: false
+        execute *KAMAL.proxy.reset_image_version, raise_on_non_zero_exit: false
+        execute *KAMAL.proxy.reset_run_command, raise_on_non_zero_exit: false
       end
     else
       raise ArgumentError, "Unknown boot_config subcommand #{subcommand}"
@@ -71,20 +117,9 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
             "Stopping and removing kamal-proxy on #{host}, if running..."
             execute *KAMAL.proxy.stop, raise_on_non_zero_exit: false
             execute *KAMAL.proxy.remove_container
+            execute *KAMAL.proxy.ensure_apps_config_directory
 
             execute *KAMAL.proxy.run
-
-            KAMAL.roles_on(host).select(&:running_proxy?).each do |role|
-              app = KAMAL.app(role: role, host: host)
-
-              version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-              endpoint = capture_with_info(*app.container_id_for_version(version)).strip
-
-              if endpoint.present?
-                info "Deploying #{endpoint} for role `#{role}` on #{host}..."
-                execute *app.deploy(target: endpoint)
-              end
-            end
           end
           run_hook "post-proxy-reboot", hosts: host_list
         end

lib/kamal/cli/server.rb

@@ -2,8 +2,10 @@ class Kamal::Cli::Server < Kamal::Cli::Base
   desc "exec", "Run a custom command on the server (use --help to show options)"
   option :interactive, type: :boolean, aliases: "-i", default: false, desc: "Run the command interactively (use for console/bash)"
   def exec(*cmd)
+    pre_connect_if_required
+
     cmd = Kamal::Utils.join_commands(cmd)
-    hosts = KAMAL.hosts | KAMAL.accessory_hosts
+    hosts = KAMAL.hosts
 
     case
     when options[:interactive]
@@ -27,7 +29,7 @@ class Kamal::Cli::Server < Kamal::Cli::Base
     with_lock do
       missing = []
 
-      on(KAMAL.hosts | KAMAL.accessory_hosts) do |host|
+      on(KAMAL.hosts) do |host|
         unless execute(*KAMAL.docker.installed?, raise_on_non_zero_exit: false)
           if execute(*KAMAL.docker.superuser?, raise_on_non_zero_exit: false)
             info "Missing Docker on #{host}. Installing…"

lib/kamal/cli/templates/sample_hooks/post-deploy.sample

@@ -7,7 +7,7 @@
 # KAMAL_PERFORMER
 # KAMAL_VERSION
 # KAMAL_HOSTS
-# KAMAL_ROLE (if set)
+# KAMAL_ROLES (if set)
 # KAMAL_DESTINATION (if set)
 # KAMAL_RUNTIME
 

lib/kamal/cli/templates/sample_hooks/pre-build.sample

@@ -13,7 +13,7 @@
 # KAMAL_PERFORMER
 # KAMAL_VERSION
 # KAMAL_HOSTS
-# KAMAL_ROLE (if set)
+# KAMAL_ROLES (if set)
 # KAMAL_DESTINATION (if set)
 
 if [ -n "$(git status --porcelain)" ]; then

lib/kamal/cli/templates/sample_hooks/pre-connect.sample

@@ -9,7 +9,7 @@
 # KAMAL_PERFORMER
 # KAMAL_VERSION
 # KAMAL_HOSTS
-# KAMAL_ROLE (if set)
+# KAMAL_ROLES (if set)
 # KAMAL_DESTINATION (if set)
 # KAMAL_RUNTIME
 

lib/kamal/cli/templates/sample_hooks/pre-deploy.sample

@@ -13,7 +13,7 @@
 # KAMAL_HOSTS
 # KAMAL_COMMAND
 # KAMAL_SUBCOMMAND
-# KAMAL_ROLE (if set)
+# KAMAL_ROLES (if set)
 # KAMAL_DESTINATION (if set)
 
 # Only check the build status for production deployments
@@ -82,11 +82,12 @@ end
 
 $stdout.sync = true
 
-puts "Checking build status..."
-attempts = 0
-checks = GithubStatusChecks.new
-
 begin
+  puts "Checking build status..."
+
+  attempts = 0
+  checks = GithubStatusChecks.new
+
   loop do
     case checks.state
     when "success"

lib/kamal/commander.rb

@@ -5,7 +5,7 @@ require "active_support/core_ext/object/blank"
 class Kamal::Commander
   attr_accessor :verbosity, :holding_lock, :connected
   attr_reader :specific_roles, :specific_hosts
-  delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :proxy_hosts, :accessory_hosts, to: :specifics
+  delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :app_hosts, :proxy_hosts, :accessory_hosts, to: :specifics
 
   def initialize
     reset
@@ -13,7 +13,7 @@ class Kamal::Commander
 
   def reset
     self.verbosity = :info
-    self.holding_lock = false
+    self.holding_lock = ENV["KAMAL_LOCK"] == "true"
     self.connected = false
     @specifics = @specific_roles = @specific_hosts = nil
     @config = @config_kwargs = nil

lib/kamal/commander/specifics.rb

@@ -11,13 +11,17 @@ class Kamal::Commander::Specifics
     @primary_role = primary_or_first_role(roles_on(primary_host))
 
     stable_sort!(roles) { |role| role == primary_role ? 0 : 1 }
-    stable_sort!(hosts) { |host| roles_on(host).any? { |role| role == primary_role } ? 0 : 1 }
+    sort_primary_role_hosts_first!(hosts)
   end
 
   def roles_on(host)
     roles.select { |role| role.hosts.include?(host.to_s) }
   end
 
+  def app_hosts
+    @app_hosts ||= sort_primary_role_hosts_first!(config.app_hosts & specified_hosts)
+  end
+
   def proxy_hosts
     config.proxy_hosts & specified_hosts
   end
@@ -51,4 +55,8 @@ class Kamal::Commander::Specifics
         specified_hosts
       end
     end
+
+    def sort_primary_role_hosts_first!(hosts)
+      stable_sort!(hosts) { |host| roles_on(host).any? { |role| role == primary_role } ? 0 : 1 }
+    end
 end

lib/kamal/commands/accessory.rb

@@ -6,7 +6,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
            :network_args, :publish_args, :env_args, :volume_args, :label_args, :option_args,
            :secrets_io, :secrets_path, :env_directory, :proxy, :running_proxy?, :registry,
            to: :accessory_config
-  delegate :proxy_container_name, to: :config
 
   def initialize(config, name:)
     super(config)
@@ -37,8 +36,8 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     docker :container, :stop, service_name
   end
 
-  def info
+  def info(all: false, quiet: false)
-    docker :ps, *service_filter
+    docker :ps, *("-a" if all), *("-q" if quiet), *service_filter
   end
 
   def logs(timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil)

lib/kamal/commands/accessory/proxy.rb

@@ -1,5 +1,5 @@
 module Kamal::Commands::Accessory::Proxy
-  delegate :proxy_container_name, to: :config
+  delegate :container_name, to: :"config.proxy_boot", prefix: :proxy
 
   def deploy(target:)
     proxy_exec :deploy, service_name, *proxy.deploy_command_args(target: target)

lib/kamal/commands/app.rb

@@ -1,5 +1,5 @@
 class Kamal::Commands::App < Kamal::Commands::Base
-  include Assets, Containers, Execution, Images, Logging, Proxy
+  include Assets, Containers, ErrorPages, Execution, Images, Logging, Proxy
 
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 

lib/kamal/commands/app/error_pages.rb

@@ -0,0 +1,9 @@
+module Kamal::Commands::App::ErrorPages
+  def create_error_pages_directory
+    make_directory(config.proxy_boot.error_pages_directory)
+  end
+
+  def clean_up_error_pages
+    [ :find, config.proxy_boot.error_pages_directory, "-mindepth", "1", "-maxdepth", "1", "!", "-name", KAMAL.config.version, "-exec", "rm", "-rf", "{} +" ]
+  end
+end

lib/kamal/commands/app/proxy.rb

@@ -1,5 +1,5 @@
 module Kamal::Commands::App::Proxy
-  delegate :proxy_container_name, to: :config
+  delegate :container_name, to: :"config.proxy_boot", prefix: :proxy
 
   def deploy(target:)
     proxy_exec :deploy, role.container_prefix, *role.proxy.deploy_command_args(target: target)
@@ -9,6 +9,18 @@ module Kamal::Commands::App::Proxy
     proxy_exec :remove, role.container_prefix
   end
 
+  def live
+    proxy_exec :resume, role.container_prefix
+  end
+
+  def maintenance(**options)
+    proxy_exec :stop, role.container_prefix, *role.proxy.stop_command_args(**options)
+  end
+
+  def remove_proxy_app_directory
+    remove_directory config.proxy_boot.app_directory
+  end
+
   private
     def proxy_exec(*command)
       docker :exec, proxy_container_name, "kamal-proxy", *command

lib/kamal/commands/auditor.rb

@@ -1,5 +1,6 @@
 class Kamal::Commands::Auditor < Kamal::Commands::Base
   attr_reader :details
+  delegate :escape_shell_value, to: Kamal::Utils
 
   def initialize(config, **details)
     super(config)
@@ -9,11 +10,8 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
   # Runs remotely
   def record(line, **details)
     combine \
-      [ :mkdir, "-p", config.run_directory ],
+      make_run_directory,
-      append(
+      append([ :echo, escape_shell_value(audit_line(line, **details)) ], audit_log_file)
-        [ :echo, audit_tags(**details).except(:version, :service_version, :service).to_s, line ],
-        audit_log_file
-      )
   end
 
   def reveal
@@ -30,4 +28,12 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
     def audit_tags(**details)
       tags(**self.details, **details)
     end
+
+    def make_run_directory
+      [ :mkdir, "-p", config.run_directory ]
+    end
+
+    def audit_line(line, **details)
+      "#{audit_tags(**details).except(:version, :service_version, :service)} #{line}"
+    end
 end

lib/kamal/commands/base.rb

@@ -68,6 +68,10 @@ module Kamal::Commands
         combine *commands, by: "||"
       end
 
+      def substitute(*commands)
+        "\$\(#{commands.join(" ")}\)"
+      end
+
       def xargs(command)
         [ :xargs, command ].flatten
       end

lib/kamal/commands/builder/base.rb

@@ -20,7 +20,8 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
       *([ "--builder", builder_name ] unless docker_driver?),
       *build_tag_options(tag_as_dirty: tag_as_dirty),
       *build_options,
-      build_context
+      build_context,
+      "2>&1"
   end
 
   def pull

lib/kamal/commands/proxy.rb

@@ -2,14 +2,7 @@ class Kamal::Commands::Proxy < Kamal::Commands::Base
   delegate :argumentize, :optionize, to: Kamal::Utils
 
   def run
-    docker :run,
+    pipe boot_config, xargs(docker_run)
-      "--name", container_name,
-      "--network", "kamal",
-      "--detach",
-      "--restart", "unless-stopped",
-      "--volume", "kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy",
-      "\$\(#{get_boot_options.join(" ")}\)",
-      config.proxy_image
   end
 
   def start
@@ -31,7 +24,7 @@ class Kamal::Commands::Proxy < Kamal::Commands::Base
   def version
     pipe \
       docker(:inspect, container_name, "--format '{{.Config.Image}}'"),
-      [ :cut, "-d:", "-f2" ]
+      [ :awk, "-F:", "'{print \$NF}'" ]
   end
 
   def logs(timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil)
@@ -65,23 +58,70 @@ class Kamal::Commands::Proxy < Kamal::Commands::Base
   end
 
   def ensure_proxy_directory
-    make_directory config.proxy_directory
+    make_directory config.proxy_boot.host_directory
   end
 
   def remove_proxy_directory
-    remove_directory config.proxy_directory
+    remove_directory config.proxy_boot.host_directory
   end
 
-  def get_boot_options
+  def ensure_apps_config_directory
-    combine [ :cat, config.proxy_options_file ], [ :echo, "\"#{config.proxy_options_default.join(" ")}\"" ], by: "||"
+    make_directory config.proxy_boot.apps_directory
+  end
+
+  def boot_config
+    [ :echo, "#{substitute(read_boot_options)} #{substitute(read_image)}:#{substitute(read_image_version)} #{substitute(read_run_command)}" ]
+  end
+
+  def read_boot_options
+    read_file(config.proxy_boot.options_file, default: config.proxy_boot.default_boot_options.join(" "))
+  end
+
+  def read_image
+    read_file(config.proxy_boot.image_file, default: config.proxy_boot.image_default)
+  end
+
+  def read_image_version
+    read_file(config.proxy_boot.image_version_file, default: Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION)
+  end
+
+  def read_run_command
+    read_file(config.proxy_boot.run_command_file)
   end
 
   def reset_boot_options
-    remove_file config.proxy_options_file
+    remove_file config.proxy_boot.options_file
+  end
+
+  def reset_image
+    remove_file config.proxy_boot.image_file
+  end
+
+  def reset_image_version
+    remove_file config.proxy_boot.image_version_file
+  end
+
+  def reset_run_command
+    remove_file config.proxy_boot.run_command_file
   end
 
   private
     def container_name
-      config.proxy_container_name
+      config.proxy_boot.container_name
+    end
+
+    def read_file(file, default: nil)
+      combine [ :cat, file, "2>", "/dev/null" ], [ :echo, "\"#{default}\"" ], by: "||"
+    end
+
+    def docker_run
+      docker \
+        :run,
+        "--name", container_name,
+        "--network", "kamal",
+        "--detach",
+        "--restart", "unless-stopped",
+        "--volume", "kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy",
+        *config.proxy_boot.apps_volume.docker_args
     end
 end

lib/kamal/configuration.rb

@@ -10,15 +10,10 @@ class Kamal::Configuration
   delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_reader :destination, :raw_config, :secrets
-  attr_reader :accessories, :aliases, :boot, :builder, :env, :logging, :proxy, :servers, :ssh, :sshkit, :registry
+  attr_reader :accessories, :aliases, :boot, :builder, :env, :logging, :proxy, :proxy_boot, :servers, :ssh, :sshkit, :registry
 
   include Validation
 
-  PROXY_MINIMUM_VERSION = "v0.8.4"
-  PROXY_HTTP_PORT = 80
-  PROXY_HTTPS_PORT = 443
-  PROXY_LOG_MAX_SIZE = "10m"
-
   class << self
     def create_from(config_file:, destination: nil, version: nil)
       ENV["KAMAL_DESTINATION"] = destination
@@ -68,7 +63,8 @@ class Kamal::Configuration
     @env = Env.new(config: @raw_config.env || {}, secrets: secrets)
 
     @logging = Logging.new(logging_config: @raw_config.logging)
-    @proxy = Proxy.new(config: self, proxy_config: @raw_config.proxy || {})
+    @proxy = Proxy.new(config: self, proxy_config: @raw_config.proxy)
+    @proxy_boot = Proxy::Boot.new(config: self)
     @ssh = Ssh.new(config: self)
     @sshkit = Sshkit.new(config: self)
 
@@ -105,6 +101,10 @@ class Kamal::Configuration
     raw_config.minimum_version
   end
 
+  def service_and_destination
+    [ service, destination ].compact.join("-")
+  end
+
   def roles
     servers.roles
   end
@@ -121,6 +121,10 @@ class Kamal::Configuration
     (roles + accessories).flat_map(&:hosts).uniq
   end
 
+  def app_hosts
+    roles.flat_map(&:hosts).uniq
+  end
+
   def primary_host
     primary_role&.primary_host
   end
@@ -145,8 +149,12 @@ class Kamal::Configuration
     proxy_roles.flat_map(&:name)
   end
 
+  def proxy_accessories
+    accessories.select(&:running_proxy?)
+  end
+
   def proxy_hosts
-    proxy_roles.flat_map(&:hosts).uniq
+    (proxy_roles.flat_map(&:hosts) + proxy_accessories.flat_map(&:hosts)).uniq
   end
 
   def repository
@@ -210,7 +218,7 @@ class Kamal::Configuration
   end
 
   def app_directory
-    File.join apps_directory, [ service, destination ].compact.join("-")
+    File.join apps_directory, service_and_destination
   end
 
   def env_directory
@@ -229,6 +237,10 @@ class Kamal::Configuration
     raw_config.asset_path
   end
 
+  def error_pages_path
+    raw_config.error_pages_path
+  end
+
   def env_tags
     @env_tags ||= if (tags = raw_config.env["tags"])
       tags.collect { |name, config| Env::Tag.new(name, config: config, secrets: secrets) }
@@ -241,42 +253,6 @@ class Kamal::Configuration
     env_tags.detect { |t| t.name == name.to_s }
   end
 
-  def proxy_publish_args(http_port, https_port, bind_ips = nil)
-    ensure_valid_bind_ips(bind_ips)
-
-    (bind_ips || [ nil ]).map do |bind_ip|
-      bind_ip = format_bind_ip(bind_ip)
-      publish_http = [ bind_ip, http_port, PROXY_HTTP_PORT ].compact.join(":")
-      publish_https = [ bind_ip, https_port, PROXY_HTTPS_PORT ].compact.join(":")
-
-      argumentize "--publish", [ publish_http, publish_https ]
-    end.join(" ")
-  end
-
-  def proxy_logging_args(max_size)
-    argumentize "--log-opt", "max-size=#{max_size}" if max_size.present?
-  end
-
-  def proxy_options_default
-    [ *proxy_publish_args(PROXY_HTTP_PORT, PROXY_HTTPS_PORT), *proxy_logging_args(PROXY_LOG_MAX_SIZE) ]
-  end
-
-  def proxy_image
-    "basecamp/kamal-proxy:#{PROXY_MINIMUM_VERSION}"
-  end
-
-  def proxy_container_name
-    "kamal-proxy"
-  end
-
-  def proxy_directory
-    File.join run_directory, "proxy"
-  end
-
-  def proxy_options_file
-    File.join proxy_directory, "options"
-  end
-
   def to_h
     {
       roles: role_names,
@@ -306,10 +282,13 @@ class Kamal::Configuration
     end
 
     def ensure_required_keys_present
-      %i[ service image registry servers ].each do |key|
+      %i[ service image registry ].each do |key|
         raise Kamal::ConfigurationError, "Missing required configuration for #{key}" unless raw_config[key].present?
       end
 
+      if raw_config.servers.nil?
+        raise Kamal::ConfigurationError, "No servers or accessories specified" unless raw_config.accessories.present?
+      else
         unless role(primary_role_name).present?
           raise Kamal::ConfigurationError, "The primary_role #{primary_role_name} isn't defined"
         end
@@ -325,6 +304,7 @@ class Kamal::Configuration
             end
           end
         end
+      end
 
       true
     end
@@ -343,15 +323,6 @@ class Kamal::Configuration
       true
     end
 
-    def ensure_valid_bind_ips(bind_ips)
-      bind_ips.present? && bind_ips.each do |ip|
-        next if ip =~ Resolv::IPv4::Regex || ip =~ Resolv::IPv6::Regex
-        raise ArgumentError, "Invalid publish IP address: #{ip}"
-      end
-
-      true
-    end
-
     def ensure_retain_containers_valid
       raise Kamal::ConfigurationError, "Must retain at least 1 container" if retain_containers < 1
 
@@ -383,15 +354,6 @@ class Kamal::Configuration
       true
     end
 
-    def format_bind_ip(ip)
-      # Ensure IPv6 address inside square brackets - e.g. [::1]
-      if ip =~ Resolv::IPv6::Regex && ip !~ /\[.*\]/
-        "[#{ip}]"
-      else
-        ip
-      end
-    end
-
     def role_names
       raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort
     end

lib/kamal/configuration/accessory.rb

@@ -32,7 +32,7 @@ class Kamal::Configuration::Accessory
   end
 
   def hosts
-    hosts_from_host || hosts_from_hosts || hosts_from_roles
+    hosts_from_host || hosts_from_hosts || hosts_from_roles || hosts_from_tags
   end
 
   def port
@@ -201,11 +201,31 @@ class Kamal::Configuration::Accessory
     end
 
     def hosts_from_roles
-      if accessory_config.key?("roles")
+      if accessory_config.key?("role")
+       config.role(accessory_config["role"])&.hosts
+      elsif accessory_config.key?("roles")
         accessory_config["roles"].flat_map { |role| config.role(role)&.hosts }
       end
     end
 
+    def hosts_from_tags
+      if accessory_config.key?("tag")
+        extract_hosts_from_config_with_tag(accessory_config["tag"])
+      elsif accessory_config.key?("tags")
+        accessory_config["tags"].flat_map { |tag| extract_hosts_from_config_with_tag(tag) }
+      end
+    end
+
+    def extract_hosts_from_config_with_tag(tag)
+      if (servers_with_roles = config.raw_config.servers).is_a?(Hash)
+        servers_with_roles.flat_map do |role, servers_in_role|
+          servers_in_role.filter_map do |host|
+            host.keys.first if host.is_a?(Hash) && host.values.first.include?(tag)
+          end
+        end
+      end
+    end
+
     def network
       accessory_config["network"] || DEFAULT_NETWORK
     end
@@ -213,6 +233,8 @@ class Kamal::Configuration::Accessory
     def ensure_valid_roles
       if accessory_config["roles"] && (missing_roles = accessory_config["roles"] - config.roles.map(&:name)).any?
         raise Kamal::ConfigurationError, "accessories/#{name}: unknown roles #{missing_roles.join(", ")}"
+      elsif accessory_config["role"] && !config.role(accessory_config["role"])
+        raise Kamal::ConfigurationError, "accessories/#{name}: unknown role #{accessory_config["role"]}"
       end
     end
 end

lib/kamal/configuration/docs/accessory.yml

@@ -46,13 +46,18 @@ accessories:
 
     # Accessory hosts
     #
-    # Specify one of `host`, `hosts`, or `roles`:
+    # Specify one of `host`, `hosts`, `role`, `roles`, `tag` or `tags`:
     host: mysql-db1
     hosts:
       - mysql-db1
       - mysql-db2
+    role: mysql
     roles:
       - mysql
+    tag: writer
+    tags:
+      - writer
+      - reader
 
     # Custom command
     #

lib/kamal/configuration/docs/configuration.yml

@@ -82,6 +82,12 @@ asset_path: /path/to/assets
 # See https://kamal-deploy.org/docs/hooks for more information:
 hooks_path: /user_home/kamal/hooks
 
+# Error pages
+#
+# A directory relative to the app root to find error pages for the proxy to serve.
+# Any files in the format 4xx.html or 5xx.html will be copied to the hosts.
+error_pages_path: public
+
 # Require destinations
 #
 # Whether deployments require a destination to be specified, defaults to `false`:

lib/kamal/configuration/docs/env.yml

@@ -51,6 +51,37 @@ env:
   secret:
     - DB_PASSWORD
 
+# Aliased secrets
+#
+# You can also alias secrets to other secrets using a `:` separator.
+#
+# This is useful when the ENV name is different from the secret name. For example, if you have two
+# places where you need to define the ENV variable `DB_PASSWORD`, but the value is different depending
+# on the context.
+#
+# ```shell
+# SECRETS=$(kamal secrets fetch ...)
+#
+# MAIN_DB_PASSWORD=$(kamal secrets extract MAIN_DB_PASSWORD $SECRETS)
+# SECONDARY_DB_PASSWORD=$(kamal secrets extract SECONDARY_DB_PASSWORD $SECRETS)
+# ```
+env:
+  secret:
+    - DB_PASSWORD:MAIN_DB_PASSWORD
+  tags:
+    secondary_db:
+      secret:
+        - DB_PASSWORD:SECONDARY_DB_PASSWORD
+accessories:
+  main_db_accessory:
+    env:
+      secret:
+        - DB_PASSWORD:MAIN_DB_PASSWORD
+  secondary_db_accessory:
+    env:
+      secret:
+        - DB_PASSWORD:SECONDARY_DB_PASSWORD
+
 # Tags
 #
 # Tags are used to add extra env variables to specific hosts.

lib/kamal/configuration/docs/proxy.yml

@@ -10,11 +10,6 @@
 # They are application-specific, so they are not shared when multiple applications
 # run on the same proxy.
 #
-# The proxy is enabled by default on the primary role but can be disabled by
-# setting `proxy: false`.
-#
-# It is disabled by default on all other roles but can be enabled by setting
-# `proxy: true` or providing a proxy configuration.
 proxy:
 
   # Hosts
@@ -52,6 +47,13 @@ proxy:
   # Defaults to `false`:
   ssl: true
 
+  # SSL redirect
+  #
+  # By default, kamal-proxy will redirect all HTTP requests to HTTPS when SSL is enabled.
+  # If you prefer that HTTP traffic is passed through to your application (along with
+  # HTTPS traffic), you can disable this redirect by setting `ssl_redirect: false`:
+  ssl_redirect: false
+
   # Forward headers
   #
   # Whether to forward the `X-Forwarded-For` and `X-Forwarded-Proto` headers.
@@ -106,3 +108,30 @@ proxy:
     response_headers:
       - X-Request-ID
       - X-Request-Start
+
+# Enabling/disabling the proxy on roles
+#
+# The proxy is enabled by default on the primary role but can be disabled by
+# setting `proxy: false` in the primary role's configuration.
+#
+# ```yaml
+# servers:
+#   web:
+#     hosts:
+#      - ...
+#     proxy: false
+# ```
+#
+# It is disabled by default on all other roles but can be enabled by setting
+# `proxy: true` or providing a proxy configuration for that role.
+#
+# ```yaml
+# servers:
+#   web:
+#     hosts:
+#      - ...
+#   web2:
+#     hosts:
+#      - ...
+#     proxy: true
+# ```

lib/kamal/configuration/env.rb

@@ -1,8 +1,7 @@
 class Kamal::Configuration::Env
   include Kamal::Configuration::Validation
 
-  attr_reader :context, :secrets
+  attr_reader :context, :clear, :secret_keys
-  attr_reader :clear, :secret_keys
   delegate :argumentize, to: Kamal::Utils
 
   def initialize(config:, secrets:, context: "env")
@@ -18,12 +17,22 @@ class Kamal::Configuration::Env
   end
 
   def secrets_io
-    Kamal::EnvFile.new(secret_keys.to_h { |key| [ key, secrets[key] ] }).to_io
+    Kamal::EnvFile.new(aliased_secrets).to_io
   end
 
   def merge(other)
     self.class.new \
       config: { "clear" => clear.merge(other.clear), "secret" => secret_keys | other.secret_keys },
-      secrets: secrets
+      secrets: @secrets
+  end
+
+  private
+    def aliased_secrets
+      secret_keys.to_h { |key| extract_alias(key) }.transform_values { |secret_key| @secrets[secret_key] }
+    end
+
+    def extract_alias(key)
+      key_name, key_aliased_to = key.split(":", 2)
+      [ key_name, key_aliased_to || key_name ]
     end
 end

lib/kamal/configuration/proxy.rb

@@ -11,6 +11,7 @@ class Kamal::Configuration::Proxy
   def initialize(config:, proxy_config:, context: "proxy")
     @config = config
     @proxy_config = proxy_config
+    @proxy_config = {} if @proxy_config.nil?
     validate! @proxy_config, with: Kamal::Configuration::Validator::Proxy, context: context
   end
 
@@ -42,8 +43,10 @@ class Kamal::Configuration::Proxy
       "max-request-body": proxy_config.dig("buffering", "max_request_body"),
       "max-response-body": proxy_config.dig("buffering", "max_response_body"),
       "forward-headers": proxy_config.dig("forward_headers"),
+      "tls-redirect": proxy_config.dig("ssl_redirect"),
       "log-request-header": proxy_config.dig("logging", "request_headers") || DEFAULT_LOG_REQUEST_HEADERS,
-      "log-response-header": proxy_config.dig("logging", "response_headers")
+      "log-response-header": proxy_config.dig("logging", "response_headers"),
+      "error-pages": error_pages
     }.compact
   end
 
@@ -51,6 +54,17 @@ class Kamal::Configuration::Proxy
     optionize ({ target: "#{target}:#{app_port}" }).merge(deploy_options), with: "="
   end
 
+  def stop_options(drain_timeout: nil, message: nil)
+    {
+      "drain-timeout": seconds_duration(drain_timeout),
+      message: message
+    }.compact
+  end
+
+  def stop_command_args(**options)
+    optionize stop_options(**options), with: "="
+  end
+
   def merge(other)
     self.class.new config: config, proxy_config: proxy_config.deep_merge(other.proxy_config)
   end
@@ -59,4 +73,8 @@ class Kamal::Configuration::Proxy
     def seconds_duration(value)
       value ? "#{value}s" : nil
     end
+
+    def error_pages
+      File.join config.proxy_boot.error_pages_container_directory, config.version if config.error_pages_path
+    end
 end

lib/kamal/configuration/proxy/boot.rb

@@ -0,0 +1,121 @@
+class Kamal::Configuration::Proxy::Boot
+  MINIMUM_VERSION = "v0.9.0"
+  DEFAULT_HTTP_PORT = 80
+  DEFAULT_HTTPS_PORT = 443
+  DEFAULT_LOG_MAX_SIZE = "10m"
+
+  attr_reader :config
+  delegate :argumentize, :optionize, to: Kamal::Utils
+
+  def initialize(config:)
+    @config = config
+  end
+
+  def publish_args(http_port, https_port, bind_ips = nil)
+    ensure_valid_bind_ips(bind_ips)
+
+    (bind_ips || [ nil ]).map do |bind_ip|
+      bind_ip = format_bind_ip(bind_ip)
+      publish_http = [ bind_ip, http_port, DEFAULT_HTTP_PORT ].compact.join(":")
+      publish_https = [ bind_ip, https_port, DEFAULT_HTTPS_PORT ].compact.join(":")
+
+      argumentize "--publish", [ publish_http, publish_https ]
+    end.join(" ")
+  end
+
+  def logging_args(max_size)
+    argumentize "--log-opt", "max-size=#{max_size}" if max_size.present?
+  end
+
+  def default_boot_options
+    [
+      *(publish_args(DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT, nil)),
+      *(logging_args(DEFAULT_LOG_MAX_SIZE))
+    ]
+  end
+
+  def repository_name
+    "basecamp"
+  end
+
+  def image_name
+    "kamal-proxy"
+  end
+
+  def image_default
+    "#{repository_name}/#{image_name}"
+  end
+
+  def container_name
+    "kamal-proxy"
+  end
+
+  def host_directory
+    File.join config.run_directory, "proxy"
+  end
+
+  def options_file
+    File.join host_directory, "options"
+  end
+
+  def image_file
+    File.join host_directory, "image"
+  end
+
+  def image_version_file
+    File.join host_directory, "image_version"
+  end
+
+  def run_command_file
+    File.join host_directory, "run_command"
+  end
+
+  def apps_directory
+    File.join host_directory, "apps-config"
+  end
+
+  def apps_container_directory
+    "/home/kamal-proxy/.apps-config"
+  end
+
+  def apps_volume
+    Kamal::Configuration::Volume.new \
+      host_path: apps_directory,
+      container_path: apps_container_directory
+  end
+
+  def app_directory
+    File.join apps_directory, config.service_and_destination
+  end
+
+  def app_container_directory
+    File.join apps_container_directory, config.service_and_destination
+  end
+
+  def error_pages_directory
+    File.join app_directory, "error_pages"
+  end
+
+  def error_pages_container_directory
+    File.join app_container_directory, "error_pages"
+  end
+
+  private
+    def ensure_valid_bind_ips(bind_ips)
+      bind_ips.present? && bind_ips.each do |ip|
+        next if ip =~ Resolv::IPv4::Regex || ip =~ Resolv::IPv6::Regex
+        raise ArgumentError, "Invalid publish IP address: #{ip}"
+      end
+
+      true
+    end
+
+    def format_bind_ip(ip)
+      # Ensure IPv6 address inside square brackets - e.g. [::1]
+      if ip =~ Resolv::IPv6::Regex && ip !~ /\A\[.*\]\z/
+        "[#{ip}]"
+      else
+        ip
+      end
+    end
+end

lib/kamal/configuration/servers.rb

@@ -13,6 +13,13 @@ class Kamal::Configuration::Servers
 
   private
     def role_names
-      servers_config.is_a?(Array) ? [ "web" ] : servers_config.keys.sort
+      case servers_config
+      when Array
+        [ "web" ]
+      when NilClass
+        []
+      else
+        servers_config.keys.sort
+      end
     end
 end

lib/kamal/configuration/validator.rb

@@ -168,4 +168,10 @@ class Kamal::Configuration::Validator
       unknown_keys.reject! { |key| extension?(key) } if allow_extensions?
       unknown_keys_error unknown_keys if unknown_keys.present?
     end
+
+    def validate_docker_options!(options)
+      if options
+        error "Cannot set restart policy in docker options, unless-stopped is required" if options["restart"]
+      end
+    end
 end

lib/kamal/configuration/validator/accessory.rb

@@ -2,8 +2,10 @@ class Kamal::Configuration::Validator::Accessory < Kamal::Configuration::Validat
   def validate!
     super
 
-    if (config.keys & [ "host", "hosts", "roles" ]).size != 1
+    if (config.keys & [ "host", "hosts", "role", "roles", "tag", "tags" ]).size != 1
-      error "specify one of `host`, `hosts` or `roles`"
+      error "specify one of `host`, `hosts`, `role`, `roles`, `tag` or `tags`"
     end
+
+    validate_docker_options!(config["options"])
   end
 end

lib/kamal/configuration/validator/role.rb

@@ -6,6 +6,7 @@ class Kamal::Configuration::Validator::Role < Kamal::Configuration::Validator
       validate_servers!(config)
     else
       super
+      validate_docker_options!(config["options"])
     end
   end
 end

lib/kamal/configuration/validator/servers.rb

@@ -1,6 +1,6 @@
 class Kamal::Configuration::Validator::Servers < Kamal::Configuration::Validator
   def validate!
-    validate_type! config, Array, Hash
+    validate_type! config, Array, Hash, NilClass
 
     validate_servers! config if config.is_a?(Array)
   end

lib/kamal/secrets/adapters/aws_secrets_manager.rb

@@ -26,6 +26,7 @@ class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Ba
     def get_from_secrets_manager(secrets, account: nil)
       args = [ "aws", "secretsmanager", "batch-get-secret-value", "--secret-id-list" ] + secrets.map(&:shellescape)
       args += [ "--profile", account.shellescape ] if account
+      args += [ "--output", "json" ]
       cmd = args.join(" ")
 
       `#{cmd}`.tap do |secrets|

lib/kamal/secrets/dotenv/inline_command_substitution.rb

@@ -4,7 +4,7 @@ class Kamal::Secrets::Dotenv::InlineCommandSubstitution
       ::Dotenv::Parser.substitutions.map! { |sub| sub == ::Dotenv::Substitutions::Command ? self : sub }
     end
 
-    def call(value, _env, overwrite: false)
+    def call(value, env, overwrite: false)
       # Process interpolated shell commands
       value.gsub(Dotenv::Substitutions::Command.singleton_class::INTERPOLATED_SHELL_COMMAND) do |*|
         # Eliminate opening and closing parentheses
@@ -14,6 +14,7 @@ class Kamal::Secrets::Dotenv::InlineCommandSubstitution
           # Command is escaped, don't replace it.
           $LAST_MATCH_INFO[0][1..]
         else
+          command = ::Dotenv::Substitutions::Variable.call(command, env)
           if command =~ /\A\s*kamal\s*secrets\s+/
             # Inline the command
             inline_secrets_command(command)

lib/kamal/version.rb

@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "2.5.2"
+  VERSION = "2.6.1"
 end

test/cli/accessory_test.rb

@@ -115,6 +115,7 @@ class CliAccessoryTest < CliTestCase
 
   test "exec" do
     run_command("exec", "mysql", "mysql -v").tap do |output|
+      assert_match "docker login private.registry -u [REDACTED] -p [REDACTED]", output
       assert_match "Launching command from new container", output
       assert_match "mysql -v", output
     end
@@ -251,6 +252,19 @@ class CliAccessoryTest < CliTestCase
     end
   end
 
+  test "boot with web role filter" do
+    run_command("boot", "redis", "-r", "web").tap do |output|
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
+    end
+  end
+
+  test "boot with workers role filter" do
+    run_command("boot", "redis", "-r", "workers").tap do |output|
+      assert_no_match "docker run", output
+    end
+  end
+
   private
     def run_command(*command)
       stdouted { Kamal::Cli::Accessory.start([ *command, "-c", "test/fixtures/deploy_with_accessories_with_different_registries.yml" ]) }

test/cli/app_test.rb

@@ -192,6 +192,34 @@ class CliAppTest < CliTestCase
     Thread.report_on_exception = true
   end
 
+  test "boot with only workers" do
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running").at_least_once # workers health check
+
+    run_command("boot", config: :with_only_workers, host: nil).tap do |output|
+      assert_match /First workers container is healthy on 1.1.1.\d, booting any other roles/, output
+      assert_no_match "kamal-proxy", output
+    end
+  end
+
+  test "boot with error pages" do
+    with_error_pages(directory: "public") do
+      stub_running
+      run_command("boot", config: :with_error_pages).tap do |output|
+        assert_match /Uploading .*kamal-error-pages.*\/latest to \.kamal\/proxy\/apps-config\/app\/error_pages/, output
+        assert_match "docker tag dhh/app:latest dhh/app:latest", output
+        assert_match /docker run --detach --restart unless-stopped --name app-web-latest --network kamal --hostname 1.1.1.1-[0-9a-f]{12} /, output
+        assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+        assert_match "Running /usr/bin/env find .kamal/proxy/apps-config/app/error_pages -mindepth 1 -maxdepth 1 ! -name latest -exec rm -rf {} + on 1.1.1.1", output
+      end
+    end
+  end
+
   test "start" do
     SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("999") # old version
 
@@ -244,9 +272,11 @@ class CliAppTest < CliTestCase
 
   test "remove" do
     run_command("remove").tap do |output|
-      assert_match /#{Regexp.escape("sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop")}/, output
+      assert_match "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop", output
-      assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output
+      assert_match "docker container prune --force --filter label=service=app", output
-      assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output
+      assert_match "docker image prune --all --force --filter label=service=app", output
+      assert_match "rm -r .kamal/apps/app on 1.1.1.1", output
+      assert_match "rm -r .kamal/proxy/apps-config/app on 1.1.1.1", output
     end
   end
 
@@ -268,12 +298,27 @@ class CliAppTest < CliTestCase
     end
   end
 
+  test "remove_app_directories" do
+    run_command("remove_app_directories").tap do |output|
+      assert_match "rm -r .kamal/apps/app on 1.1.1.1", output
+      assert_match "rm -r .kamal/proxy/apps-config/app on 1.1.1.1", output
+    end
+  end
+
   test "exec" do
     run_command("exec", "ruby -v").tap do |output|
+      assert_match "docker login -u [REDACTED] -p [REDACTED]", output
       assert_match "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:latest ruby -v", output
     end
   end
 
+  test "exec without command fails" do
+    error = assert_raises(ArgumentError, "Exec requires a command to be specified") do
+      run_command("exec")
+    end
+    assert_equal "No command provided. You must specify a command to execute.", error.message
+  end
+
   test "exec separate arguments" do
     run_command("exec", "ruby", " -v").tap do |output|
       assert_match "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:latest ruby -v", output
@@ -312,18 +357,25 @@ class CliAppTest < CliTestCase
   end
 
   test "exec interactive" do
+    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     SSHKit::Backend::Abstract.any_instance.expects(:exec)
       .with("ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:latest ruby -v'")
+
     run_command("exec", "-i", "ruby -v").tap do |output|
+      assert_hook_ran "pre-connect", output
+      assert_match "docker login -u [REDACTED] -p [REDACTED]", output
       assert_match "Get most recent version available as an image...", output
       assert_match "Launching interactive command with version latest via SSH from new container on 1.1.1.1...", output
     end
   end
 
   test "exec interactive with reuse" do
+    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     SSHKit::Backend::Abstract.any_instance.expects(:exec)
       .with("ssh -t root@1.1.1.1 -p 22 'docker exec -it app-web-999 ruby -v'")
+
     run_command("exec", "-i", "--reuse", "ruby -v").tap do |output|
+      assert_hook_ran "pre-connect", output
       assert_match "Get current version of running container...", output
       assert_match "Running /usr/bin/env sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done on 1.1.1.1", output
       assert_match "Launching interactive command with version 999 via SSH from existing container on 1.1.1.1...", output
@@ -437,6 +489,24 @@ class CliAppTest < CliTestCase
     end
   end
 
+  test "live" do
+    run_command("live").tap do |output|
+      assert_match "docker exec kamal-proxy kamal-proxy resume app-web on 1.1.1.1", output
+    end
+  end
+
+  test "maintenance" do
+    run_command("maintenance").tap do |output|
+      assert_match "docker exec kamal-proxy kamal-proxy stop app-web --drain-timeout=\"30s\" on 1.1.1.1", output
+    end
+  end
+
+  test "maintenance with options" do
+    run_command("maintenance", "--message", "Hello", "--drain_timeout", "10").tap do |output|
+      assert_match "docker exec kamal-proxy kamal-proxy stop app-web --drain-timeout=\"10s\" --message=\"Hello\" on 1.1.1.1", output
+    end
+  end
+
   private
     def run_command(*command, config: :with_accessories, host: "1.1.1.1", allow_execute_error: false)
       stdouted do

test/cli/build_test.rb

@@ -21,11 +21,12 @@ class CliBuildTest < CliTestCase
         .returns("")
 
       run_command("push", "--verbose").tap do |output|
+        assert_hook_ran "pre-connect", output
         assert_hook_ran "pre-build", output
         assert_match /Cloning repo into build directory/, output
         assert_match /git -C #{Dir.tmpdir}\/kamal-clones\/app-#{pwd_sha} clone #{Dir.pwd}/, output
         assert_match /docker --version && docker buildx version/, output
-        assert_match /docker buildx build --output=type=registry --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
+        assert_match /docker buildx build --output=type=registry --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. 2>&1 as .*@localhost/, output
       end
     end
   end
@@ -47,7 +48,7 @@ class CliBuildTest < CliTestCase
         assert_match /Cloning repo into build directory/, output
         assert_match /git -C #{Dir.tmpdir}\/kamal-clones\/app-#{pwd_sha} clone #{Dir.pwd}/, output
         assert_match /docker --version && docker buildx version/, output
-        assert_match /docker buildx build --output=type=docker --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
+        assert_match /docker buildx build --output=type=docker --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. 2>&1 as .*@localhost/, output
       end
     end
   end
@@ -57,6 +58,7 @@ class CliBuildTest < CliTestCase
       stub_setup
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with { |*args| args[0..1] == [ :docker, :login ] }
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
         .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd, "--recurse-submodules")
@@ -70,7 +72,7 @@ class CliBuildTest < CliTestCase
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :submodule, :update, "--init")
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
-        .with(:docker, :buildx, :build, "--output=type=registry", "--platform", "linux/amd64", "--builder", "kamal-local-docker-container", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".")
+        .with(:docker, :buildx, :build, "--output=type=registry", "--platform", "linux/amd64", "--builder", "kamal-local-docker-container", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".", "2>&1")
 
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
         .with(:git, "-C", anything, :"rev-parse", :HEAD)
@@ -94,7 +96,7 @@ class CliBuildTest < CliTestCase
       assert_no_match /Cloning repo into build directory/, output
       assert_hook_ran "pre-build", output
       assert_match /docker --version && docker buildx version/, output
-      assert_match /docker buildx build --output=type=registry --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile . as .*@localhost/, output
+      assert_match /docker buildx build --output=type=registry --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile . 2>&1 as .*@localhost/, output
     end
   end
 
@@ -103,6 +105,7 @@ class CliBuildTest < CliTestCase
       stub_setup
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with { |*args| args[0..1] == [ :docker, :login ] }
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
         .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd, "--recurse-submodules")
@@ -139,6 +142,9 @@ class CliBuildTest < CliTestCase
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
         .with(:docker, "--version", "&&", :docker, :buildx, "version")
 
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| args[0..1] == [ :docker, :login ] }
+
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
         .with(:docker, :buildx, :rm, "kamal-local-docker-container")
 
@@ -160,7 +166,7 @@ class CliBuildTest < CliTestCase
         .returns("")
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
-        .with(:docker, :buildx, :build, "--output=type=registry", "--platform", "linux/amd64", "--builder", "kamal-local-docker-container", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".")
+        .with(:docker, :buildx, :build, "--output=type=registry", "--platform", "linux/amd64", "--builder", "kamal-local-docker-container", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".", "2>&1")
 
       run_command("push").tap do |output|
         assert_match /WARN Missing compatible builder, so creating a new one first/, output
@@ -302,7 +308,7 @@ class CliBuildTest < CliTestCase
       run_command("dev", "--verbose").tap do |output|
         assert_no_match(/Cloning repo into build directory/, output)
         assert_match(/docker --version && docker buildx version/, output)
-        assert_match(/docker buildx build --output=type=docker --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999-dirty -t dhh\/app:latest-dirty --label service="app" --file Dockerfile \. as .*@localhost/, output)
+        assert_match(/docker buildx build --output=type=docker --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999-dirty -t dhh\/app:latest-dirty --label service="app" --file Dockerfile \. 2>&1 as .*@localhost/, output)
       end
     end
   end
@@ -314,14 +320,14 @@ class CliBuildTest < CliTestCase
       run_command("dev", "--output=local", "--verbose").tap do |output|
         assert_no_match(/Cloning repo into build directory/, output)
         assert_match(/docker --version && docker buildx version/, output)
-        assert_match(/docker buildx build --output=type=local --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999-dirty -t dhh\/app:latest-dirty --label service="app" --file Dockerfile \. as .*@localhost/, output)
+        assert_match(/docker buildx build --output=type=local --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999-dirty -t dhh\/app:latest-dirty --label service="app" --file Dockerfile \. 2>&1 as .*@localhost/, output)
       end
     end
   end
 
   private
     def run_command(*command, fixture: :with_accessories)
-      stdouted { Kamal::Cli::Build.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) }
+      stdouted { stderred { Kamal::Cli::Build.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) } }
     end
 
     def stub_dependency_checks

test/cli/main_test.rb

@@ -21,7 +21,6 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
     # deploy
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -32,7 +31,6 @@ class CliMainTest < CliTestCase
       assert_match /Ensure Docker is installed.../, output
       # deploy
       assert_match /Acquiring the deploy lock/, output
-      assert_match /Log into image registry/, output
       assert_match /Pull app image/, output
       assert_match /Ensure kamal-proxy is running/, output
       assert_match /Detect stale containers/, output
@@ -45,7 +43,6 @@ class CliMainTest < CliTestCase
     with_test_secrets("secrets" => "DB_PASSWORD=secret") do
       invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true }
 
-      Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
       Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
       Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
       Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -56,7 +53,6 @@ class CliMainTest < CliTestCase
 
       run_command("deploy", "--verbose").tap do |output|
         assert_hook_ran "pre-connect", output
-        assert_match /Log into image registry/, output
         assert_match /Build and push app image/, output
         assert_hook_ran "pre-deploy", output
         assert_match /Ensure kamal-proxy is running/, output
@@ -70,7 +66,6 @@ class CliMainTest < CliTestCase
   test "deploy with skip_push" do
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -79,7 +74,6 @@ class CliMainTest < CliTestCase
 
     run_command("deploy", "--skip_push").tap do |output|
       assert_match /Acquiring the deploy lock/, output
-      assert_match /Log into image registry/, output
       assert_match /Pull app image/, output
       assert_match /Ensure kamal-proxy is running/, output
       assert_match /Detect stale containers/, output
@@ -122,6 +116,32 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "deploy when inheriting lock" do
+    Thread.report_on_exception = false
+
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
+
+    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+
+    with_kamal_lock_env do
+      KAMAL.reset
+      run_command("deploy").tap do |output|
+        assert_no_match /Acquiring the deploy lock/, output
+        assert_match /Build and push app image/, output
+        assert_match /Ensure kamal-proxy is running/, output
+        assert_match /Detect stale containers/, output
+        assert_match /Prune old containers and images/, output
+        assert_no_match /Releasing the deploy lock/, output
+      end
+    end
+  end
+
   test "deploy error when locking" do
     Thread.report_on_exception = false
 
@@ -153,11 +173,11 @@ class CliMainTest < CliTestCase
     end
   end
 
-  test "deploy errors during outside section leave remove lock" do
+  test "deploy errors during outside section leave remote lock" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, :skip_local => false }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Kamal::Cli::Main.any_instance.expects(:invoke)
-      .with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
+      .with("kamal:cli:build:deliver", [], invoke_options)
       .raises(RuntimeError)
 
     assert_not KAMAL.holding_lock?
@@ -170,7 +190,6 @@ class CliMainTest < CliTestCase
   test "deploy with skipped hooks" do
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => true }
 
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -185,7 +204,6 @@ class CliMainTest < CliTestCase
   test "deploy with missing secrets" do
     invoke_options = { "config_file" => "test/fixtures/deploy_with_secrets.yml", "version" => "999", "skip_hooks" => false }
 
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -570,4 +588,11 @@ class CliMainTest < CliTestCase
     def assert_file(file, content)
       assert_match content, File.read(file)
     end
+
+    def with_kamal_lock_env
+      ENV["KAMAL_LOCK"] = "true"
+      yield
+    ensure
+      ENV.delete("KAMAL_LOCK")
+    end
 end

test/cli/proxy_test.rb

@@ -4,25 +4,26 @@ class CliProxyTest < CliTestCase
   test "boot" do
     run_command("boot").tap do |output|
       assert_match "docker login", output
-      assert_match "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image}", output
+      assert_match "mkdir -p .kamal/proxy/apps-config", output
+      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
     end
   end
 
   test "boot old version" do
     Thread.report_on_exception = false
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2")
+      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :awk, "-F:", "'{print $NF}'")
       .returns("v0.0.1")
       .at_least_once
 
     exception = assert_raises do
       run_command("boot").tap do |output|
         assert_match "docker login", output
-        assert_match "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image}", output
+        assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
       end
     end
 
-    assert_includes exception.message, "kamal-proxy version v0.0.1 is too old, run `kamal proxy reboot` in order to update to at least #{Kamal::Configuration::PROXY_MINIMUM_VERSION}"
+    assert_includes exception.message, "kamal-proxy version v0.0.1 is too old, run `kamal proxy reboot` in order to update to at least #{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}"
   ensure
     Thread.report_on_exception = false
   end
@@ -30,53 +31,33 @@ class CliProxyTest < CliTestCase
   test "boot correct version" do
     Thread.report_on_exception = false
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2")
+      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :awk, "-F:", "'{print $NF}'")
-      .returns(Kamal::Configuration::PROXY_MINIMUM_VERSION)
+      .returns(Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION)
       .at_least_once
 
     run_command("boot").tap do |output|
       assert_match "docker login", output
-      assert_match "docker container start kamal-proxy || docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image}", output
+      assert_match "docker container start kamal-proxy || echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
     end
   ensure
     Thread.report_on_exception = false
   end
 
   test "reboot" do
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet")
-      .returns("abcdefabcdef")
-      .at_least_once
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with { |*args| args[0..1] == [ :sh, "-c" ] }
-      .returns("123")
-      .at_least_once
-
     run_command("reboot", "-y").tap do |output|
       assert_match "docker container stop kamal-proxy on 1.1.1.1", output
       assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.1", output
-      assert_match "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image} on 1.1.1.1", output
+      assert_match "mkdir -p .kamal/proxy/apps-config on 1.1.1.1", output
-      assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target=\"abcdefabcdef:80\" --deploy-timeout=\"6s\" --drain-timeout=\"30s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\" on 1.1.1.1", output
+      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config on 1.1.1.1", output
 
       assert_match "docker container stop kamal-proxy on 1.1.1.2", output
       assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.2", output
-      assert_match "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image} on 1.1.1.2", output
+      assert_match "mkdir -p .kamal/proxy/apps-config on 1.1.1.1", output
-      assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target=\"abcdefabcdef:80\" --deploy-timeout=\"6s\" --drain-timeout=\"30s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\" on 1.1.1.2", output
+      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config on 1.1.1.2", output
     end
   end
 
   test "reboot --rolling" do
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet")
-      .returns("abcdefabcdef")
-      .at_least_once
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with { |*args| args[0..1] == [ :sh, "-c" ] }
-      .returns("123")
-      .at_least_once
-
     run_command("reboot", "--rolling", "-y").tap do |output|
       assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.1", output
     end
@@ -179,8 +160,8 @@ class CliProxyTest < CliTestCase
     SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("12345678")
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2")
+      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :awk, "-F:", "'{print $NF}'")
-      .returns(Kamal::Configuration::PROXY_MINIMUM_VERSION)
+      .returns(Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION)
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
@@ -196,7 +177,7 @@ class CliProxyTest < CliTestCase
       assert_match "/usr/bin/env mkdir -p .kamal", output
       assert_match "docker network create kamal", output
       assert_match "docker login -u [REDACTED] -p [REDACTED]", output
-      assert_match "docker container start kamal-proxy || docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION}", output
+      assert_match "docker container start kamal-proxy || echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
       assert_match "/usr/bin/env mkdir -p .kamal", output
       assert_match %r{docker rename app-web-latest app-web-latest_replaced_.*}, output
       assert_match "/usr/bin/env mkdir -p .kamal/apps/app/env/roles", output
@@ -218,8 +199,8 @@ class CliProxyTest < CliTestCase
     SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("12345678")
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2")
+      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :awk, "-F:", "'{print $NF}'")
-      .returns(Kamal::Configuration::PROXY_MINIMUM_VERSION)
+      .returns(Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION)
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
@@ -238,7 +219,10 @@ class CliProxyTest < CliTestCase
     run_command("boot_config", "set").tap do |output|
       %w[ 1.1.1.1 1.1.1.2 ].each do |host|
         assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
-        assert_match "Uploading \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
       end
     end
   end
@@ -248,6 +232,9 @@ class CliProxyTest < CliTestCase
       %w[ 1.1.1.1 1.1.1.2 ].each do |host|
         assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
         assert_match "Uploading \"--log-opt max-size=10m\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
       end
     end
   end
@@ -257,6 +244,9 @@ class CliProxyTest < CliTestCase
       %w[ 1.1.1.1 1.1.1.2 ].each do |host|
         assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
         assert_match "Uploading \"--publish 80:80 --publish 443:443 --log-opt max-size=100m\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
       end
     end
   end
@@ -266,6 +256,9 @@ class CliProxyTest < CliTestCase
       %w[ 1.1.1.1 1.1.1.2 ].each do |host|
         assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
         assert_match "Uploading \"--publish 80:80 --publish 443:443\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
       end
     end
   end
@@ -310,19 +303,81 @@ class CliProxyTest < CliTestCase
       %w[ 1.1.1.1 1.1.1.2 ].each do |host|
         assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
         assert_match "Uploading \"--publish 80:80 --publish 443:443 --log-opt max-size=10m --label=foo=bar --add_host=thishost:thathost\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
+      end
+    end
+  end
+
+  test "boot_config set registry" do
+    run_command("boot_config", "set", "--registry", "myreg").tap do |output|
+      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
+        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/options on #{host}", output
+        assert_match "Uploading \"myreg/basecamp/kamal-proxy\" to .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
+      end
+    end
+  end
+
+  test "boot_config set repository" do
+    run_command("boot_config", "set", "--repository", "myrepo").tap do |output|
+      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
+        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/options on #{host}", output
+        assert_match "Uploading \"myrepo/kamal-proxy\" to .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
+      end
+    end
+  end
+
+  test "boot_config set image_version" do
+    run_command("boot_config", "set", "--image_version", "0.9.9").tap do |output|
+      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
+        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Uploading \"0.9.9\" to .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
+      end
+    end
+  end
+
+  test "boot_config set run_command" do
+    run_command("boot_config", "set", "--metrics_port", "9000", "--debug", "true").tap do |output|
+      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
+        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
+        assert_match "Uploading \"--publish 80:80 --publish 443:443 --log-opt max-size=10m --expose=9000\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Uploading \"kamal-proxy run --debug --metrics-port \\\"9000\\\"\" to .kamal/proxy/run_command on #{host}", output
+      end
+    end
+  end
+
+  test "boot_config set all" do
+    run_command("boot_config", "set", "--docker_options", "label=foo=bar", "--registry", "myreg", "--repository", "myrepo", "--image_version", "0.9.9", "--metrics_port", "9000", "--debug", "true").tap do |output|
+      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
+        assert_match "Uploading \"--publish 80:80 --publish 443:443 --log-opt max-size=10m --expose=9000 --label=foo=bar\" to .kamal/proxy/options on #{host}", output
+        assert_match "Uploading \"myreg/myrepo/kamal-proxy\" to .kamal/proxy/image on #{host}", output
+        assert_match "Uploading \"0.9.9\" to .kamal/proxy/image_version on #{host}", output
+        assert_match "Uploading \"kamal-proxy run --debug --metrics-port \\\"9000\\\"\" to .kamal/proxy/run_command on #{host}", output
       end
     end
   end
 
   test "boot_config get" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:cat, ".kamal/proxy/options", "||", :echo, "\"--publish 80:80 --publish 443:443 --log-opt max-size=10m\"")
+      .with(:echo, "$(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\")")
-      .returns("--publish 80:80 --publish 8443:443 --label=foo=bar")
+      .returns("--publish 80:80 --publish 8443:443 --label=foo=bar basecamp/kamal-proxy:v1.0.0")
       .twice
 
     run_command("boot_config", "get").tap do |output|
-      assert_match "Host 1.1.1.1: --publish 80:80 --publish 8443:443 --label=foo=bar", output
+      assert_match "Host 1.1.1.1: --publish 80:80 --publish 8443:443 --label=foo=bar basecamp/kamal-proxy:v1.0.0", output
-      assert_match "Host 1.1.1.2: --publish 80:80 --publish 8443:443 --label=foo=bar", output
+      assert_match "Host 1.1.1.2: --publish 80:80 --publish 8443:443 --label=foo=bar basecamp/kamal-proxy:v1.0.0", output
     end
   end
 

test/commander_test.rb

@@ -149,6 +149,12 @@ class CommanderTest < ActiveSupport::TestCase
     assert_equal [], @kamal.accessory_hosts
   end
 
+  test "primary role hosts are first" do
+    configure_with(:deploy_with_roles_workers_primary)
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @kamal.hosts
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @kamal.app_hosts
+  end
+
   private
     def configure_with(variant)
       @kamal = Kamal::Commander.new.tap do |kamal|

test/commands/app_test.rb

@@ -497,6 +497,30 @@ class CommandsAppTest < ActiveSupport::TestCase
     ], new_command(asset_path: "/public/assets").clean_up_assets
   end
 
+  test "live" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy resume app-web",
+      new_command.live.join(" ")
+  end
+
+  test "maintenance" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy stop app-web",
+      new_command.maintenance.join(" ")
+  end
+
+  test "maintenance with options" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy stop app-web --drain-timeout=\"10s\" --message=\"Hi\"",
+      new_command.maintenance(drain_timeout: 10, message: "Hi").join(" ")
+  end
+
+  test "remove_proxy_app_directory" do
+    assert_equal \
+      "rm -r .kamal/proxy/apps-config/app",
+      new_command.remove_proxy_app_directory.join(" ")
+  end
+
   private
     def new_command(role: "web", host: "1.1.1.1", **additional_config)
       config = Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999")

test/commands/auditor_test.rb

@@ -20,8 +20,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
     assert_equal [
       :mkdir, "-p", ".kamal", "&&",
       :echo,
-      "[#{@recorded_at}] [#{@performer}]",
+      "\"[#{@recorded_at}] [#{@performer}] app removed container\"",
-      "app removed container",
       ">>", ".kamal/app-audit.log"
     ], @auditor.record("app removed container")
   end
@@ -31,8 +30,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
       assert_equal [
         :mkdir, "-p", ".kamal", "&&",
         :echo,
-        "[#{@recorded_at}] [#{@performer}] [staging]",
+        "\"[#{@recorded_at}] [#{@performer}] [staging] app removed container\"",
-        "app removed container",
         ">>", ".kamal/app-staging-audit.log"
       ], auditor.record("app removed container")
     end
@@ -43,8 +41,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
       assert_equal [
         :mkdir, "-p", ".kamal", "&&",
         :echo,
-        "[#{@recorded_at}] [#{@performer}] [web]",
+        "\"[#{@recorded_at}] [#{@performer}] [web] app removed container\"",
-        "app removed container",
         ">>", ".kamal/app-audit.log"
       ], auditor.record("app removed container")
     end
@@ -54,8 +51,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
     assert_equal [
       :mkdir, "-p", ".kamal", "&&",
       :echo,
-      "[#{@recorded_at}] [#{@performer}] [value]",
+      "\"[#{@recorded_at}] [#{@performer}] [value] app removed container\"",
-      "app removed container",
       ">>", ".kamal/app-audit.log"
     ], @auditor.record("app removed container", detail: "value")
   end

test/commands/builder_test.rb

@@ -9,7 +9,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "cache" => { "type" => "gha" } })
     assert_equal "local", builder.name
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile . 2>&1",
       builder.push.join(" ")
   end
 
@@ -17,7 +17,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "arch" => [ "amd64" ] })
     assert_equal "local", builder.name
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile . 2>&1",
       builder.push.join(" ")
   end
 
@@ -25,7 +25,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "cache" => { "type" => "gha" } })
     assert_equal "local", builder.name
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile . 2>&1",
       builder.push.join(" ")
   end
 
@@ -33,7 +33,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "arch" => [ "amd64", "arm64" ], "remote" => "ssh://app@127.0.0.1", "cache" => { "type" => "gha" } })
     assert_equal "hybrid", builder.name
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64,linux/arm64 --builder kamal-hybrid-docker-container-ssh---app-127-0-0-1 -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/amd64,linux/arm64 --builder kamal-hybrid-docker-container-ssh---app-127-0-0-1 -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile . 2>&1",
       builder.push.join(" ")
   end
 
@@ -41,7 +41,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "arch" => [ "amd64", "arm64" ], "remote" => "ssh://app@127.0.0.1", "cache" => { "type" => "gha" }, "local" => false })
     assert_equal "remote", builder.name
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64,linux/arm64 --builder kamal-remote-ssh---app-127-0-0-1 -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/amd64,linux/arm64 --builder kamal-remote-ssh---app-127-0-0-1 -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile . 2>&1",
       builder.push.join(" ")
   end
 
@@ -49,7 +49,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "arch" => [ "#{remote_arch}" ], "remote" => "ssh://app@host", "cache" => { "type" => "gha" } })
     assert_equal "remote", builder.name
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/#{remote_arch} --builder kamal-remote-ssh---app-host -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/#{remote_arch} --builder kamal-remote-ssh---app-host -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile . 2>&1",
       builder.push.join(" ")
   end
 
@@ -57,7 +57,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "arch" => [ "#{local_arch}" ], "remote" => "ssh://app@host", "cache" => { "type" => "gha" } })
     assert_equal "local", builder.name
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/#{local_arch} --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/#{local_arch} --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile . 2>&1",
       builder.push.join(" ")
   end
 
@@ -65,7 +65,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     builder = new_builder_command(builder: { "arch" => [ "#{local_arch}" ], "driver" => "cloud docker-org-name/builder-name" })
     assert_equal "cloud", builder.name
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/#{local_arch} --builder cloud-docker-org-name-builder-name -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/#{local_arch} --builder cloud-docker-org-name-builder-name -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile . 2>&1",
       builder.push.join(" ")
   end
 
@@ -112,14 +112,14 @@ class CommandsBuilderTest < ActiveSupport::TestCase
   test "build context" do
     builder = new_builder_command(builder: { "context" => ".." })
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ..",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .. 2>&1",
       builder.push.join(" ")
   end
 
   test "push with build args" do
     builder = new_builder_command(builder: { "args" => { "a" => 1, "b" => 2 } })
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile . 2>&1",
       builder.push.join(" ")
   end
 
@@ -128,7 +128,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
       FileUtils.touch("Dockerfile")
       builder = new_builder_command(builder: { "secrets" => [ "a", "b" ] })
       assert_equal \
-        "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile .",
+        "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile . 2>&1",
         builder.push.join(" ")
     end
   end
@@ -148,35 +148,35 @@ class CommandsBuilderTest < ActiveSupport::TestCase
   test "context build" do
     builder = new_builder_command(builder: { "context" => "./foo" })
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo 2>&1",
       builder.push.join(" ")
   end
 
   test "push with provenance" do
     builder = new_builder_command(builder: { "provenance" => "mode=max" })
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --provenance mode=max .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --provenance mode=max . 2>&1",
       builder.push.join(" ")
   end
 
   test "push with provenance false" do
     builder = new_builder_command(builder: { "provenance" => false })
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --provenance false .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --provenance false . 2>&1",
       builder.push.join(" ")
   end
 
   test "push with sbom" do
     builder = new_builder_command(builder: { "sbom" => true })
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --sbom true .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --sbom true . 2>&1",
       builder.push.join(" ")
   end
 
   test "push with sbom false" do
     builder = new_builder_command(builder: { "sbom" => false })
     assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --sbom false .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --sbom false . 2>&1",
       builder.push.join(" ")
   end
 

test/commands/proxy_test.rb

@@ -15,7 +15,7 @@ class CommandsProxyTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION}",
+      "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config",
       new_command.run.join(" ")
   end
 
@@ -23,7 +23,7 @@ class CommandsProxyTest < ActiveSupport::TestCase
     @config.delete(:proxy)
 
     assert_equal \
-      "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION}",
+      "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config",
       new_command.run.join(" ")
   end
 
@@ -101,7 +101,7 @@ class CommandsProxyTest < ActiveSupport::TestCase
 
   test "version" do
     assert_equal \
-      "docker inspect kamal-proxy --format '{{.Config.Image}}' | cut -d: -f2",
+      "docker inspect kamal-proxy --format '{{.Config.Image}}' | awk -F: '{print $NF}'",
       new_command.version.join(" ")
   end
 
@@ -111,10 +111,28 @@ class CommandsProxyTest < ActiveSupport::TestCase
       new_command.ensure_proxy_directory.join(" ")
   end
 
-  test "get_boot_options" do
+  test "read_boot_options" do
     assert_equal \
-      "cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\"",
+      "cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\"",
-      new_command.get_boot_options.join(" ")
+      new_command.read_boot_options.join(" ")
+  end
+
+  test "read_image" do
+    assert_equal \
+      "cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"",
+      new_command.read_image.join(" ")
+  end
+
+  test "read_image_version" do
+    assert_equal \
+      "cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\"",
+      new_command.read_image_version.join(" ")
+  end
+
+  test "read_run_command" do
+    assert_equal \
+      "cat .kamal/proxy/run_command 2> /dev/null || echo \"\"",
+      new_command.read_run_command.join(" ")
   end
 
   test "reset_boot_options" do
@@ -123,6 +141,30 @@ class CommandsProxyTest < ActiveSupport::TestCase
       new_command.reset_boot_options.join(" ")
   end
 
+  test "reset_image" do
+    assert_equal \
+      "rm .kamal/proxy/image",
+      new_command.reset_image.join(" ")
+  end
+
+  test "reset_image_version" do
+    assert_equal \
+      "rm .kamal/proxy/image_version",
+      new_command.reset_image_version.join(" ")
+  end
+
+  test "ensure_apps_config_directory" do
+    assert_equal \
+      "mkdir -p .kamal/proxy/apps-config",
+      new_command.ensure_apps_config_directory.join(" ")
+  end
+
+  test "reset_run_command" do
+    assert_equal \
+      "rm .kamal/proxy/run_command",
+      new_command.reset_run_command.join(" ")
+  end
+
   private
     def new_command
       Kamal::Commands::Proxy.new(Kamal::Configuration.new(@config, version: "123"))

test/configuration/accessory_test.rb

@@ -7,8 +7,8 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
       image: "dhh/app",
       registry: { "username" => "dhh", "password" => "secret" },
       servers: {
-        "web" => [ "1.1.1.1", "1.1.1.2" ],
+        "web" => [ { "1.1.1.1" => "writer" }, { "1.1.1.2" => "reader" } ],
-        "workers" => [ "1.1.1.3", "1.1.1.4" ]
+        "workers" => [ { "1.1.1.3" => "writer" }, "1.1.1.4" ]
       },
       builder: { "arch" => "amd64" },
       env: { "REDIS_URL" => "redis://x/y" },
@@ -55,7 +55,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
           "service" => "custom-monitoring",
           "image" => "monitoring:latest",
           "registry" => { "server" => "other.registry", "username" => "user", "password" => "pw" },
-          "roles" => [ "web" ],
+          "role" => "web",
           "port" => "4321:4321",
           "labels" => {
             "cache" => "true"
@@ -70,6 +70,14 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
           "proxy" => {
             "host" => "monitoring.example.com"
           }
+        },
+        "proxy" => {
+          "image" => "proxy:latest",
+          "tags" => [ "writer", "reader" ]
+        },
+        "logger" => {
+          "image" => "logger:latest",
+          "tag" => "writer"
         }
       }
     }
@@ -107,6 +115,8 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
     assert_equal [ "1.1.1.5" ], @config.accessory(:mysql).hosts
     assert_equal [ "1.1.1.6", "1.1.1.7" ], @config.accessory(:redis).hosts
     assert_equal [ "1.1.1.1", "1.1.1.2" ], @config.accessory(:monitoring).hosts
+    assert_equal [ "1.1.1.1", "1.1.1.3", "1.1.1.2" ], @config.accessory(:proxy).hosts
+    assert_equal [ "1.1.1.1", "1.1.1.3" ], @config.accessory(:logger).hosts
   end
 
   test "missing host" do
@@ -117,14 +127,14 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
     end
   end
 
-  test "setting host, hosts and roles" do
+  test "setting host, hosts, roles and tags" do
     @deploy[:accessories]["mysql"]["hosts"] = [ "mysql-db1" ]
     @deploy[:accessories]["mysql"]["roles"] = [ "db" ]
 
     exception = assert_raises(Kamal::ConfigurationError) do
       Kamal::Configuration.new(@deploy)
     end
-    assert_equal "accessories/mysql: specify one of `host`, `hosts` or `roles`", exception.message
+    assert_equal "accessories/mysql: specify one of `host`, `hosts`, `role`, `roles`, `tag` or `tags`", exception.message
   end
 
   test "all hosts" do
@@ -187,4 +197,12 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
     assert @config.accessory(:monitoring).running_proxy?
     assert_equal [ "monitoring.example.com" ], @config.accessory(:monitoring).proxy.hosts
   end
+
+  test "can't set restart in options" do
+    @deploy[:accessories]["mysql"]["options"] = { "restart" => "always" }
+
+    assert_raises Kamal::ConfigurationError, "servers/workers: Cannot set restart policy in docker options, unless-stopped is required" do
+      Kamal::Configuration.new(@deploy)
+    end
+  end
 end

test/configuration/env/tags_test.rb

@@ -92,7 +92,25 @@ class ConfigurationEnvTagsTest < ActiveSupport::TestCase
       }
 
       config = Kamal::Configuration.new(deploy)
-      assert_equal "hello", config.role("web").env("1.1.1.1").secrets["PASSWORD"]
+      assert_equal "PASSWORD=hello\n", config.role("web").env("1.1.1.1").secrets_io.string
+    end
+  end
+
+  test "aliased tag secret env" do
+    with_test_secrets("secrets" => "PASSWORD=hello\nALIASED_PASSWORD=aliased_hello") do
+      deploy = {
+        service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+        servers: [ { "1.1.1.1" => "secrets" } ],
+        builder: { "arch" => "amd64" },
+        env: {
+          "tags" => {
+            "secrets" => { "secret" => [ "PASSWORD:ALIASED_PASSWORD" ] }
+          }
+        }
+      }
+
+      config = Kamal::Configuration.new(deploy)
+      assert_equal "PASSWORD=aliased_hello\n", config.role("web").env("1.1.1.1").secrets_io.string
     end
   end
 

test/configuration/env_test.rb

@@ -48,6 +48,20 @@ class ConfigurationEnvTest < ActiveSupport::TestCase
     end
   end
 
+  test "aliased secrets" do
+    with_test_secrets("secrets" => "ALIASED_PASSWORD=hello") do
+      config = {
+        "secret" => [ "PASSWORD:ALIASED_PASSWORD" ],
+        "clear" => {}
+      }
+
+      assert_config \
+        config: config,
+        clear: {},
+        secrets: { "PASSWORD" => "hello" }
+    end
+  end
+
   private
     def assert_config(config:, clear: {}, secrets: {})
       env = Kamal::Configuration::Env.new config: config, secrets: Kamal::Secrets.new

test/configuration/proxy/boot_test.rb

@@ -0,0 +1,29 @@
+require "test_helper"
+
+class ConfigurationProxyBootTest < ActiveSupport::TestCase
+  setup do
+    ENV["RAILS_MASTER_KEY"] = "456"
+    ENV["VERSION"] = "missing"
+
+    @deploy = {
+      service: "app", image: "dhh/app",
+      registry: { "username" => "dhh", "password" => "secret" },
+      builder: { "arch" => "amd64" },
+      env: { "REDIS_URL" => "redis://x/y" },
+      servers: [ "1.1.1.1", "1.1.1.2" ],
+      volumes: [ "/local/path:/container/path" ]
+    }
+
+    @config = Kamal::Configuration.new(@deploy)
+    @proxy_boot_config = @config.proxy_boot
+  end
+
+  test "proxy directories" do
+    assert_equal ".kamal/proxy/apps-config", @proxy_boot_config.apps_directory
+    assert_equal "/home/kamal-proxy/.apps-config", @proxy_boot_config.apps_container_directory
+    assert_equal ".kamal/proxy/apps-config/app", @proxy_boot_config.app_directory
+    assert_equal "/home/kamal-proxy/.apps-config/app", @proxy_boot_config.app_container_directory
+    assert_equal ".kamal/proxy/apps-config/app/error_pages", @proxy_boot_config.error_pages_directory
+    assert_equal "/home/kamal-proxy/.apps-config/app/error_pages", @proxy_boot_config.error_pages_container_directory
+  end
+end

test/configuration/proxy_test.rb

@@ -38,6 +38,13 @@ class ConfigurationProxyTest < ActiveSupport::TestCase
     assert_not config.proxy.ssl?
   end
 
+  test "false not allowed" do
+    @deploy[:proxy] = false
+    assert_raises(Kamal::ConfigurationError, "proxy: should be a hash") do
+      config.proxy
+    end
+  end
+
   private
     def config
       Kamal::Configuration.new(@deploy)

test/configuration/role_test.rb

@@ -258,6 +258,14 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     assert_equal "18s", config_with_roles.role(:workers).proxy.deploy_options[:"target-timeout"]
   end
 
+  test "can't set restart in options" do
+    @deploy_with_roles[:servers]["workers"]["options"] = { "restart" => "always" }
+
+    assert_raises Kamal::ConfigurationError, "servers/workers: Cannot set restart policy in docker options, unless-stopped is required" do
+      Kamal::Configuration.new(@deploy_with_roles)
+    end
+  end
+
   private
     def config
       Kamal::Configuration.new(@deploy)

test/configuration_test.rb

@@ -30,7 +30,7 @@ class ConfigurationTest < ActiveSupport::TestCase
   %i[ service image registry ].each do |key|
     test "#{key} config required" do
       assert_raise(Kamal::ConfigurationError) do
-        Kamal::Configuration.new @deploy.tap { _1.delete key }
+        Kamal::Configuration.new @deploy.tap { |config| config.delete key }
       end
     end
   end
@@ -38,21 +38,36 @@ class ConfigurationTest < ActiveSupport::TestCase
   %w[ username password ].each do |key|
     test "registry #{key} required" do
       assert_raise(Kamal::ConfigurationError) do
-        Kamal::Configuration.new @deploy.tap { _1[:registry].delete key }
+        Kamal::Configuration.new @deploy.tap { |config| config[:registry].delete key }
       end
     end
   end
 
   test "service name valid" do
     assert_nothing_raised do
-      Kamal::Configuration.new(@deploy.tap { _1[:service] = "hey-app1_primary" })
+      Kamal::Configuration.new(@deploy.tap { |config| config[:service] = "hey-app1_primary" })
-      Kamal::Configuration.new(@deploy.tap { _1[:service] = "MyApp" })
+      Kamal::Configuration.new(@deploy.tap { |config| config[:service] = "MyApp" })
     end
   end
 
   test "service name invalid" do
     assert_raise(Kamal::ConfigurationError) do
-      Kamal::Configuration.new @deploy.tap { _1[:service] = "app.com" }
+      Kamal::Configuration.new @deploy.tap { |config| config[:service] = "app.com" }
+    end
+  end
+
+  test "servers required" do
+    assert_raise(Kamal::ConfigurationError) do
+      Kamal::Configuration.new @deploy.tap { |config| config.delete(:servers) }
+    end
+  end
+
+  test "servers not required with accessories" do
+    assert_nothing_raised do
+      @deploy.delete(:servers)
+      @deploy[:accessories] = { "foo" => { "image" => "foo/bar", "host" => "1.1.1.1" } }
+
+      Kamal::Configuration.new(@deploy)
     end
   end
 
@@ -250,7 +265,7 @@ class ConfigurationTest < ActiveSupport::TestCase
   test "destination required" do
     dest_config_file = Pathname.new(File.expand_path("fixtures/deploy_for_required_dest.yml", __dir__))
 
-    assert_raises(Kamal::ConfigurationError) do
+    assert_raises(ArgumentError, "You must specify a destination") do
       config = Kamal::Configuration.create_from config_file: dest_config_file
     end
 

test/fixtures/deploy_with_error_pages.yml

@@ -0,0 +1,11 @@
+service: app
+image: dhh/app
+servers:
+  - "1.1.1.1"
+  - "1.1.1.2"
+registry:
+  username: user
+  password: pw
+builder:
+  arch: amd64
+error_pages_path: public

test/fixtures/deploy_with_roles_workers_primary.yml

@@ -0,0 +1,19 @@
+service: app
+image: dhh/app
+servers:
+  workers:
+    - 1.1.1.1
+    - 1.1.1.2
+  web:
+    - 1.1.1.3
+    - 1.1.1.4
+env:
+  REDIS_URL: redis://x/y
+registry:
+  server: registry.digitalocean.com
+  username: user
+  password: pw
+builder:
+  arch: amd64
+deploy_timeout: 1
+primary_role: workers

test/integration/accessory_test.rb

@@ -17,10 +17,39 @@ class AccessoryTest < IntegrationTest
     logs = kamal :accessory, :logs, :busybox, capture: true
     assert_match /Starting busybox.../, logs
 
+    boot = kamal :accessory, :boot, :busybox, capture: true
+    assert_match /Skipping booting `busybox` on vm1, vm2, a container already exists/, boot
+
     kamal :accessory, :remove, :busybox, "-y"
     assert_accessory_not_running :busybox
   end
 
+  test "proxied: boot, stop, start, restart, logs, remove" do
+    @app = "app_with_proxied_accessory"
+
+    kamal :proxy, :boot
+
+    kamal :accessory, :boot, :netcat
+    assert_accessory_running :netcat
+    assert_netcat_is_up
+
+    kamal :accessory, :stop, :netcat
+    assert_accessory_not_running :netcat
+    assert_netcat_not_found
+
+    kamal :accessory, :start, :netcat
+    assert_accessory_running :netcat
+    assert_netcat_is_up
+
+    kamal :accessory, :restart, :netcat
+    assert_accessory_running :netcat
+    assert_netcat_is_up
+
+    kamal :accessory, :remove, :netcat, "-y"
+    assert_accessory_not_running :netcat
+    assert_netcat_not_found
+  end
+
   private
     def assert_accessory_running(name)
       assert_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
@@ -33,4 +62,25 @@ class AccessoryTest < IntegrationTest
     def accessory_details(name)
       kamal :accessory, :details, name, capture: true
     end
+
+    def assert_netcat_is_up
+      response = netcat_response
+      debug_response_code(response, "200")
+      assert_equal "200", response.code
+    end
+
+    def assert_netcat_not_found
+      response = netcat_response
+      debug_response_code(response, "404")
+      assert_equal "404", response.code
+    end
+
+    def netcat_response
+      uri = URI.parse("http://127.0.0.1:12345/up")
+      http = Net::HTTP.new(uri.host, uri.port)
+      request = Net::HTTP::Get.new(uri)
+      request["Host"] = "netcat"
+
+      http.request(request)
+    end
 end

test/integration/app_test.rb

@@ -48,9 +48,36 @@ class AppTest < IntegrationTest
     assert_match "App Host: vm1", exec_output
     assert_match /1 root      0:\d\d nginx/, exec_output
 
+    kamal :app, :maintenance
+    assert_app_in_maintenance
+
+    kamal :app, :live
+    assert_app_is_up
+
     kamal :app, :remove
 
     assert_app_not_found
     assert_app_directory_removed
   end
+
+  test "custom error pages" do
+    @app = "app_with_roles"
+
+    kamal :deploy
+    assert_app_is_up
+
+    kamal :app, :maintenance
+    assert_app_in_maintenance message: "Custom Maintenance Page"
+
+    kamal :app, :live
+    kamal :app, :maintenance, "--message", "\"Testing Maintence Mode\""
+    assert_app_in_maintenance message: "Custom Maintenance Page: Testing Maintence Mode"
+
+    second_version = update_app_rev
+
+    kamal :redeploy
+
+    kamal :app, :maintenance
+    assert_app_in_maintenance message: "Custom Maintenance Page"
+  end
 end

test/integration/docker/deployer/app/.kamal/hooks/pre-connect

@@ -1,4 +1,5 @@
 #!/bin/sh
 
 echo "About to lock..."
+env
 mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-connect

test/integration/docker/deployer/app/.kamal/hooks/pre-deploy

@@ -1,3 +1,6 @@
 #!/bin/sh
+set -e
+
+kamal proxy boot_config set --registry registry:4443
 echo "Deployed!"
 mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-deploy

test/integration/docker/deployer/app/config/deploy.yml

@@ -41,3 +41,8 @@ accessories:
     cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done'
     roles:
       - web
+  busybox2:
+    service: custom-busybox
+    image: registry:4443/busybox:1.36.0
+    cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done'
+    host: vm3

test/integration/docker/deployer/app_with_proxied_accessory/.kamal/hooks/pre-deploy

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -e
+
+kamal proxy boot_config set --registry registry:4443

test/integration/docker/deployer/app_with_proxied_accessory/config/deploy.yml

@@ -1,7 +1,5 @@
 service: app_with_proxied_accessory
 image: app_with_proxied_accessory
-servers:
-  - vm1
 env:
   clear:
     CLEAR_TOKEN: 4321
@@ -24,15 +22,13 @@ accessories:
     service: custom-busybox
     image: registry:4443/busybox:1.36.0
     cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done'
-    roles:
+    host: vm1
-      - web
   netcat:
     service: netcat
     image: registry:4443/busybox:1.36.0
     cmd: >
       sh -c 'echo "Starting netcat..."; while true; do echo -e "HTTP/1.1 200 OK\r\nContent-Length: 11\r\n\r\nHello Ruby" | nc -l -p 80; done'
-    roles:
+    host: vm1
-      - web
     port: 12345:80
     proxy:
       host: netcat
@@ -41,4 +37,4 @@ accessories:
         interval: 1
         timeout: 1
         path: "/"
-
+drain_timeout: 2

test/integration/docker/deployer/app_with_roles/.kamal/hooks/pre-deploy

@@ -1,3 +1,6 @@
 #!/bin/sh
+set -e
+
+kamal proxy boot_config set --registry registry:4443
 echo "Deployed!"
 mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-deploy

test/integration/docker/deployer/app_with_roles/config/deploy.yml

@@ -37,6 +37,7 @@ proxy:
       - X-Request-Start
 
 asset_path: /usr/share/nginx/html/versions
+error_pages_path: error_pages
 
 registry:
   server: registry:4443

test/integration/docker/deployer/app_with_roles/error_pages/503.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>503 Service Interrupted</title>
+  </head>
+  <body>
+    <p>Custom Maintenance Page: {{ .Message }}</p>
+  </body>
+</html>

test/integration/docker/deployer/app_with_traefik/.kamal/hooks/pre-deploy

@@ -1,3 +1,7 @@
-kamal proxy boot_config set --publish false \
+set -e
+
+kamal proxy boot_config set --registry registry:4443 \
+                            --publish false \
                             --docker_options label=traefik.http.services.kamal_proxy.loadbalancer.server.scheme=http \
-                                             label=traefik.http.routers.kamal_proxy.rule=PathPrefix\(\`/\`\)
+                                             label=traefik.http.routers.kamal_proxy.rule=PathPrefix\(\`/\`\) \
+                                             sysctl=net.ipv4.ip_local_port_range=\"10000\ 60999\"

test/integration/docker/deployer/setup.sh

@@ -20,6 +20,7 @@ push_image_to_registry_4443() {
 install_kamal
 push_image_to_registry_4443 nginx 1-alpine-slim
 push_image_to_registry_4443 busybox 1.36.0
+push_image_to_registry_4443 basecamp/kamal-proxy v0.9.0
 
 # .ssh is on a shared volume that persists between runs. Clean it up as the
 # churn of temporary vm IPs can eventually create conflicts.

test/integration/docker/registry/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry
+FROM registry:3
 
 COPY boot.sh .
 

test/integration/docker/registry/boot.sh

@@ -2,4 +2,4 @@
 
 while [ ! -f /certs/domain.crt ]; do sleep 1; done
 
-exec /entrypoint.sh /etc/docker/registry/config.yml
+exec /entrypoint.sh /etc/distribution/config.yml

test/integration/integration_test.rb

@@ -11,7 +11,7 @@ class IntegrationTest < ActiveSupport::TestCase
   end
 
   teardown do
-    unless passed?
+    if !passed? && ENV["DEBUG_CONTAINER_LOGS"]
       [ :deployer, :vm1, :vm2, :shared, :load_balancer, :registry ].each do |container|
         puts
         puts "Logs for #{container}:"
@@ -25,8 +25,8 @@ class IntegrationTest < ActiveSupport::TestCase
     def docker_compose(*commands, capture: false, raise_on_error: true)
       command = "TEST_ID=#{ENV["TEST_ID"]} docker compose #{commands.join(" ")}"
       succeeded = false
-      if capture
+      if capture || !ENV["DEBUG"]
-        result = stdouted { succeeded = system("cd test/integration && #{command}") }
+        result = stdouted { stderred { succeeded = system("cd test/integration && #{command}") } }
       else
         succeeded = system("cd test/integration && #{command}")
       end
@@ -45,15 +45,22 @@ class IntegrationTest < ActiveSupport::TestCase
     end
 
     def assert_app_is_down
-      response = app_response
+      assert_app_error_code("502")
-      debug_response_code(response, "502")
+    end
-      assert_equal "502", response.code
+
+    def assert_app_in_maintenance(message: nil)
+      assert_app_error_code("503", message: message)
     end
 
     def assert_app_not_found
+      assert_app_error_code("404")
+    end
+
+    def assert_app_error_code(code, message: nil)
       response = app_response
-      debug_response_code(response, "404")
+      debug_response_code(response, code)
-      assert_equal "404", response.code
+      assert_equal code, response.code
+      assert_match message, response.body.strip if message
     end
 
     def assert_app_is_up(version: nil, app: @app)

test/integration/main_test.rb

@@ -9,8 +9,12 @@ class MainTest < IntegrationTest
     kamal :deploy
     assert_app_is_up version: first_version
     assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "pre-app-boot", "post-app-boot", "post-deploy"
+
     assert_envs version: first_version
 
+    output = kamal :app, :exec, "--verbose", "ls", "-r", "web", capture: true
+    assert_hook_env_variables output, version: first_version
+
     second_version = update_app_rev
 
     kamal :redeploy
@@ -28,7 +32,7 @@ class MainTest < IntegrationTest
     assert_match /Proxy Host: vm2/, details
     assert_match /App Host: vm1/, details
     assert_match /App Host: vm2/, details
-    assert_match /basecamp\/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION}/, details
+    assert_match /basecamp\/kamal-proxy:#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}/, details
     assert_match /registry:4443\/app:#{first_version}/, details
 
     audit = kamal :audit, capture: true
@@ -60,7 +64,7 @@ class MainTest < IntegrationTest
     version = latest_app_version
 
     assert_equal [ "web" ], config[:roles]
-    assert_equal [ "vm1", "vm2" ], config[:hosts]
+    assert_equal [ "vm1", "vm2", "vm3" ], config[:hosts]
     assert_equal "vm1", config[:primary_host]
     assert_equal version, config[:version]
     assert_equal "registry:4443/app", config[:repository]
@@ -88,8 +92,6 @@ class MainTest < IntegrationTest
   end
 
   test "setup and remove" do
-    @app = "app_with_roles"
-
     kamal :proxy, :boot_config, "set",
       "--publish=false",
       "--docker-options=label=traefik.http.services.kamal_proxy.loadbalancer.server.scheme=http",
@@ -172,21 +174,36 @@ class MainTest < IntegrationTest
       assert_equal "200", Net::HTTP.get_response(URI.parse("http://#{app_host}:12345/versions/.hidden")).code
     end
 
-    def vm1_image_ids
+    def image_ids(vm:)
-      docker_compose("exec vm1 docker image ls -q", capture: true).strip.split("\n")
+      docker_compose("exec #{vm} docker image ls -q", capture: true).strip.split("\n")
     end
 
-    def vm1_container_ids
+    def container_ids(vm:)
-      docker_compose("exec vm1 docker ps -a -q", capture: true).strip.split("\n")
+      docker_compose("exec #{vm} docker ps -a -q", capture: true).strip.split("\n")
     end
 
     def assert_no_images_or_containers
-      assert vm1_image_ids.empty?
+      [ :vm1, :vm2, :vm3 ].each do |vm|
-      assert vm1_container_ids.empty?
+        assert image_ids(vm: vm).empty?
+        assert container_ids(vm: vm).empty?
+      end
     end
 
     def assert_images_and_containers
-      assert vm1_image_ids.any?
+      [ :vm1, :vm2, :vm3 ].each do |vm|
-      assert vm1_container_ids.any?
+        assert image_ids(vm: vm).any?
+        assert container_ids(vm: vm).any?
+      end
+    end
+
+    def assert_hook_env_variables(output, version:)
+      assert_match "KAMAL_VERSION=#{version}", output
+      assert_match "KAMAL_SERVICE=app", output
+      assert_match "KAMAL_SERVICE_VERSION=app@#{version[0..6]}", output
+      assert_match "KAMAL_COMMAND=app", output
+      assert_match "KAMAL_PERFORMER=deployer@example.com", output
+      assert_match /KAMAL_RECORDED_AT=\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ/, output
+      assert_match "KAMAL_HOSTS=vm1,vm2", output
+      assert_match "KAMAL_ROLES=web", output
     end
 end

test/integration/proxied_accessory_test.rb

@@ -1,63 +0,0 @@
-require_relative "integration_test"
-
-class ProxiedAccessoryTest < IntegrationTest
-  test "boot, stop, start, restart, logs, remove" do
-    @app = "app_with_proxied_accessory"
-
-    kamal :deploy
-
-    kamal :accessory, :boot, :netcat
-    assert_accessory_running :netcat
-    assert_netcat_is_up
-
-    kamal :accessory, :stop, :netcat
-    assert_accessory_not_running :netcat
-    assert_netcat_not_found
-
-    kamal :accessory, :start, :netcat
-    assert_accessory_running :netcat
-    assert_netcat_is_up
-
-    kamal :accessory, :restart, :netcat
-    assert_accessory_running :netcat
-    assert_netcat_is_up
-
-    kamal :accessory, :remove, :netcat, "-y"
-    assert_accessory_not_running :netcat
-    assert_netcat_not_found
-  end
-
-  private
-    def assert_accessory_running(name)
-      assert_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
-    end
-
-    def assert_accessory_not_running(name)
-      assert_no_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
-    end
-
-    def accessory_details(name)
-      kamal :accessory, :details, name, capture: true
-    end
-
-    def assert_netcat_is_up
-      response = netcat_response
-      debug_response_code(response, "200")
-      assert_equal "200", response.code
-    end
-
-    def assert_netcat_not_found
-      response = netcat_response
-      debug_response_code(response, "404")
-      assert_equal "404", response.code
-    end
-
-    def netcat_response
-      uri = URI.parse("http://127.0.0.1:12345/up")
-      http = Net::HTTP.new(uri.host, uri.port)
-      request = Net::HTTP::Get.new(uri)
-      request["Host"] = "netcat"
-
-      http.request(request)
-    end
-end

test/integration/proxy_test.rb

@@ -6,6 +6,8 @@ class ProxyTest < IntegrationTest
   end
 
   test "boot, reboot, stop, start, restart, logs, remove" do
+    kamal :proxy, :boot_config, :set, "--registry", "registry:4443"
+
     kamal :proxy, :boot
     assert_proxy_running
 
@@ -46,7 +48,27 @@ class ProxyTest < IntegrationTest
     logs = kamal :proxy, :logs, capture: true
     assert_match /No previous state to restore/, logs
 
+    kamal :proxy, :boot_config, :set, "--registry", "registry:4443", "--docker-options='sysctl net.ipv4.ip_local_port_range=\"10000 60999\"'"
+    assert_docker_options_in_file
+
+    kamal :proxy, :reboot, "-y"
+    assert_docker_options_in_container
+
+    kamal :proxy, :boot_config, :reset
+
     kamal :proxy, :remove
     assert_proxy_not_running
   end
+
+  private
+    def assert_docker_options_in_file
+      boot_config = kamal :proxy, :boot_config, :get, capture: true
+      assert_match "Host vm1: --publish 80:80 --publish 443:443 --log-opt max-size=10m --sysctl net.ipv4.ip_local_port_range=\"10000 60999\"", boot_config
+    end
+
+    def assert_docker_options_in_container
+      assert_equal \
+        "{\"net.ipv4.ip_local_port_range\":\"10000 60999\"}",
+        docker_compose("exec vm1 docker inspect --format '{{ json .HostConfig.Sysctls }}' kamal-proxy", capture: true).strip
+    end
 end

test/secrets/aws_secrets_manager_adapter_test.rb

@@ -4,7 +4,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
   test "fails when errors are present" do
     stub_ticks.with("aws --version 2> /dev/null")
     stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list unknown1 unknown2 --profile default")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list unknown1 unknown2 --profile default --output json")
       .returns(<<~JSON)
         {
           "SecretValues": [],
@@ -33,7 +33,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
   test "fetch" do
     stub_ticks.with("aws --version 2> /dev/null")
     stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 secret2/KEY3 --profile default")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 secret2/KEY3 --profile default --output json")
       .returns(<<~JSON)
         {
           "SecretValues": [
@@ -76,7 +76,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
   test "fetch with string value" do
     stub_ticks.with("aws --version 2> /dev/null")
     stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret secret2/KEY1 --profile default")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret secret2/KEY1 --profile default --output json")
       .returns(<<~JSON)
         {
           "SecretValues": [
@@ -118,7 +118,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
   test "fetch with secret names" do
     stub_ticks.with("aws --version 2> /dev/null")
     stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 --profile default")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 --profile default --output json")
       .returns(<<~JSON)
         {
           "SecretValues": [
@@ -159,7 +159,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
   test "fetch without account option omits --profile" do
     stub_ticks.with("aws --version 2> /dev/null")
     stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 --output json")
       .returns(<<~JSON)
         {
           "SecretValues": [

test/test_helper.rb

@@ -58,9 +58,7 @@ class ActiveSupport::TestCase
     def setup_test_secrets(**files)
       @original_pwd = Dir.pwd
       @secrets_tmpdir = Dir.mktmpdir
-      fixtures_dup = File.join(@secrets_tmpdir, "test")
+      copy_fixtures(@secrets_tmpdir)
-      FileUtils.mkdir_p(fixtures_dup)
-      FileUtils.cp_r("test/fixtures/", fixtures_dup)
 
       Dir.chdir(@secrets_tmpdir)
       FileUtils.mkdir_p(".kamal")
@@ -75,6 +73,30 @@ class ActiveSupport::TestCase
       Dir.chdir(@original_pwd)
       FileUtils.rm_rf(@secrets_tmpdir)
     end
+
+    def with_error_pages(directory:)
+      error_pages_tmpdir = Dir.mktmpdir
+
+      Dir.mktmpdir do |tmpdir|
+        copy_fixtures(tmpdir)
+
+        Dir.chdir(tmpdir) do
+          FileUtils.mkdir_p(directory)
+          Dir.chdir(directory) do
+            File.write("404.html", "404 page")
+            File.write("503.html", "503 page")
+          end
+
+          yield
+        end
+      end
+    end
+
+    def copy_fixtures(to_dir)
+      new_test_dir = File.join(to_dir, "test")
+      FileUtils.mkdir_p(new_test_dir)
+      FileUtils.cp_r("test/fixtures/", new_test_dir)
+    end
 end
 
 class SecretAdapterTestCase < ActiveSupport::TestCase