Bump version for 2.6.1

Merge pull request #1558 from basecamp/per-role-proxy-docs
Per role proxy docs
2025-05-15 15:15:29 +01:00 · 2025-05-15 15:00:11 +01:00 · 2025-05-15 14:57:58 +01:00 · 2025-05-15 14:47:46 +01:00 · 2025-05-15 14:45:19 +01:00 · 2025-05-15 14:33:05 +01:00
84 changed files with 1354 additions and 415 deletions
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,13 +1,13 @@
 PATH
  remote: .
  specs:
-    kamal (2.5.3)
+    kamal (2.6.1)
      activesupport (>= 7.0)
      base64 (~> 0.2)
      bcrypt_pbkdf (~> 1.0)
      concurrent-ruby (~> 1.2)
      dotenv (~> 3.1)
-      ed25519 (~> 1.2)
+      ed25519 (~> 1.4)
      net-ssh (~> 7.3)
      sshkit (>= 1.23.0, < 2.0)
      thor (~> 1.3)
@@ -60,7 +60,7 @@ GEM
      reline (>= 0.3.8)
    dotenv (3.1.5)
    drb (2.2.1)
-    ed25519 (1.3.0)
+    ed25519 (1.4.0)
    erubi (1.13.0)
    i18n (1.14.6)
      concurrent-ruby (~> 1.0)
@@ -82,15 +82,15 @@ GEM
    net-sftp (4.0.0)
      net-ssh (>= 5.0.0, < 8.0.0)
    net-ssh (7.3.0)
-    nokogiri (1.18.3-aarch64-linux-musl)
+    nokogiri (1.18.8-aarch64-linux-musl)
      racc (~> 1.4)
-    nokogiri (1.18.3-arm64-darwin)
+    nokogiri (1.18.8-arm64-darwin)
      racc (~> 1.4)
-    nokogiri (1.18.3-x86_64-darwin)
+    nokogiri (1.18.8-x86_64-darwin)
      racc (~> 1.4)
-    nokogiri (1.18.3-x86_64-linux-gnu)
+    nokogiri (1.18.8-x86_64-linux-gnu)
      racc (~> 1.4)
-    nokogiri (1.18.3-x86_64-linux-musl)
+    nokogiri (1.18.8-x86_64-linux-musl)
      racc (~> 1.4)
    ostruct (0.6.1)
    parallel (1.26.3)
@@ -101,7 +101,7 @@ GEM
      date
      stringio
    racc (1.8.1)
-    rack (3.1.10)
+    rack (3.1.12)
    rack-session (2.0.0)
      rack (>= 3.0.0)
    rack-test (2.1.0)
@@ -174,7 +174,7 @@ GEM
    unicode-display_width (3.1.2)
      unicode-emoji (~> 4.0, >= 4.0.4)
    unicode-emoji (4.0.4)
-    uri (1.0.2)
+    uri (1.0.3)
    useragent (0.16.11)
    zeitwerk (2.7.1)

--- a/kamal.gemspec
+++ b/kamal.gemspec
@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
  spec.add_dependency "thor", "~> 1.3"
  spec.add_dependency "dotenv", "~> 3.1"
  spec.add_dependency "zeitwerk", ">= 2.6.18", "< 3.0"
-  spec.add_dependency "ed25519", "~> 1.2"
+  spec.add_dependency "ed25519", "~> 1.4"
  spec.add_dependency "bcrypt_pbkdf", "~> 1.0"
  spec.add_dependency "concurrent-ruby", "~> 1.2"
  spec.add_dependency "base64", "~> 0.2"
--- a/lib/kamal/cli/accessory.rb
+++ b/lib/kamal/cli/accessory.rb
@@ -1,4 +1,5 @@
 require "active_support/core_ext/array/conversions"
+require "concurrent/array"

 class Kamal::Cli::Accessory < Kamal::Cli::Base
  desc "boot [NAME]", "Boot new accessory service on host (use NAME=all to boot all accessories)"
@@ -10,6 +11,16 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
        prepare(name) if prepare

        with_accessory(name) do |accessory, hosts|
+          booted_hosts = Concurrent::Array.new
+          on(hosts) do |host|
+            booted_hosts << host.to_s if capture_with_info(*accessory.info(all: true, quiet: true)).strip.presence
+          end
+
+          if booted_hosts.any?
+            say "Skipping booting `#{name}` on #{booted_hosts.sort.join(", ")}, a container already exists", :yellow
+            hosts -= booted_hosts
+          end
+
          directories(name)
          upload(name)

@@ -130,6 +141,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
  option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
  option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
  def exec(name, *cmd)
+    pre_connect_if_required
+
    cmd = Kamal::Utils.join_commands(cmd)
    with_accessory(name) do |accessory, hosts|
      case
@@ -139,6 +152,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base

      when options[:interactive]
        say "Launching interactive command via SSH from new container...", :magenta
+        on(accessory.hosts.first) { execute *KAMAL.registry.login }
        run_locally { exec accessory.execute_in_new_container_over_ssh(cmd) }

      when options[:reuse]
@@ -151,6 +165,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
      else
        say "Launching command from new container...", :magenta
        on(hosts) do |host|
+          execute *KAMAL.registry.login
          execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
          puts_by_host host, capture_with_info(*accessory.execute_in_new_container(cmd))
        end
@@ -275,11 +290,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
    end

    def accessory_hosts(accessory)
-      if KAMAL.specific_hosts&.any?
-        KAMAL.specific_hosts & accessory.hosts
-      else
-        accessory.hosts
-      end
+      KAMAL.accessory_hosts & accessory.hosts
    end

    def remove_accessory(name)
--- a/lib/kamal/cli/app.rb
+++ b/lib/kamal/cli/app.rb
@@ -7,9 +7,11 @@ class Kamal::Cli::App < Kamal::Cli::Base
        say "Start container with version #{version} (or reboot if already running)...", :magenta

        # Assets are prepared in a separate step to ensure they are on all hosts before booting
-        on(KAMAL.hosts) do
+        on(KAMAL.app_hosts) do
+          Kamal::Cli::App::ErrorPages.new(host, self).run
+
          KAMAL.roles_on(host).each do |role|
-            Kamal::Cli::App::PrepareAssets.new(host, role, self).run
+            Kamal::Cli::App::Assets.new(host, role, self).run
          end
        end

@@ -31,7 +33,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
        end

        # Tag once the app booted on all hosts
-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
          execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
          execute *KAMAL.app.tag_latest_image
        end
@@ -42,7 +44,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
  desc "start", "Start existing app container on servers"
  def start
    with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
        roles = KAMAL.roles_on(host)

        roles.each do |role|
@@ -65,7 +67,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
  desc "stop", "Stop app container on servers"
  def stop
    with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
        roles = KAMAL.roles_on(host)

        roles.each do |role|
@@ -89,7 +91,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
  # FIXME: Drop in favor of just containers?
  desc "details", "Show details about app containers"
  def details
-    on(KAMAL.hosts) do |host|
+    on(KAMAL.app_hosts) do |host|
      roles = KAMAL.roles_on(host)

      roles.each do |role|
@@ -104,10 +106,16 @@ class Kamal::Cli::App < Kamal::Cli::Base
  option :env, aliases: "-e", type: :hash, desc: "Set environment variables for the command"
  option :detach, type: :boolean, default: false, desc: "Execute command in a detached container"
  def exec(*cmd)
+    pre_connect_if_required
+
    if (incompatible_options = [ :interactive, :reuse ].select { |key| options[:detach] && options[key] }.presence)
      raise ArgumentError, "Detach is not compatible with #{incompatible_options.join(" or ")}"
    end

+    if cmd.empty?
+      raise ArgumentError, "No command provided. You must specify a command to execute."
+    end
+
    cmd = Kamal::Utils.join_commands(cmd)
    env = options[:env]
    detach = options[:detach]
@@ -123,6 +131,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
      say "Get most recent version available as an image...", :magenta unless options[:version]
      using_version(version_or_latest) do |version|
        say "Launching interactive command with version #{version} via SSH from new container on #{KAMAL.primary_host}...", :magenta
+        on(KAMAL.primary_host) { execute *KAMAL.registry.login }
        run_locally do
          exec KAMAL.app(role: KAMAL.primary_role, host: KAMAL.primary_host).execute_in_new_container_over_ssh(cmd, env: env)
        end
@@ -133,7 +142,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
      using_version(options[:version] || current_running_version) do |version|
        say "Launching command with version #{version} from existing container...", :magenta

-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
          roles = KAMAL.roles_on(host)

          roles.each do |role|
@@ -147,7 +156,9 @@ class Kamal::Cli::App < Kamal::Cli::Base
      say "Get most recent version available as an image...", :magenta unless options[:version]
      using_version(version_or_latest) do |version|
        say "Launching command with version #{version} from new container...", :magenta
-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
+          execute *KAMAL.registry.login
+
          roles = KAMAL.roles_on(host)

          roles.each do |role|
@@ -161,7 +172,7 @@ class Kamal::Cli::App < Kamal::Cli::Base

  desc "containers", "Show app containers on servers"
  def containers
-    on(KAMAL.hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_containers) }
+    on(KAMAL.app_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_containers) }
  end

  desc "stale_containers", "Detect app stale containers"
@@ -170,7 +181,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
    stop = options[:stop]

    with_lock_if_stopping do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
        roles = KAMAL.roles_on(host)

        roles.each do |role|
@@ -193,7 +204,7 @@ class Kamal::Cli::App < Kamal::Cli::Base

  desc "images", "Show app images on servers"
  def images
-    on(KAMAL.hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_images) }
+    on(KAMAL.app_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_images) }
  end

  desc "logs", "Show log lines from app on servers (use --help to show options)"
@@ -229,7 +240,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
    else
      lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set

-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
        roles = KAMAL.roles_on(host)

        roles.each do |role|
@@ -249,14 +260,44 @@ class Kamal::Cli::App < Kamal::Cli::Base
      stop
      remove_containers
      remove_images
-      remove_app_directory
+      remove_app_directories
+    end
+  end
+
+  desc "live", "Set the app to live mode"
+  def live
+    with_lock do
+      on(KAMAL.proxy_hosts) do |host|
+        roles = KAMAL.roles_on(host)
+
+        roles.each do |role|
+          execute *KAMAL.app(role: role, host: host).live if role.running_proxy?
+        end
+      end
+    end
+  end
+
+  desc "maintenance", "Set the app to maintenance mode"
+  option :drain_timeout, type: :numeric, desc: "How long to allow in-flight requests to complete (defaults to drain_timeout from config)"
+  option :message, type: :string, desc: "Message to display to clients while stopped"
+  def maintenance
+    maintenance_options = { drain_timeout: options[:drain_timeout] || KAMAL.config.drain_timeout, message: options[:message] }
+
+    with_lock do
+      on(KAMAL.proxy_hosts) do |host|
+        roles = KAMAL.roles_on(host)
+
+        roles.each do |role|
+          execute *KAMAL.app(role: role, host: host).maintenance(**maintenance_options) if role.running_proxy?
+        end
+      end
    end
  end

  desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true
  def remove_container(version)
    with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
        roles = KAMAL.roles_on(host)

        roles.each do |role|
@@ -270,7 +311,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
  desc "remove_containers", "Remove all app containers from servers", hide: true
  def remove_containers
    with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
        roles = KAMAL.roles_on(host)

        roles.each do |role|
@@ -284,30 +325,33 @@ class Kamal::Cli::App < Kamal::Cli::Base
  desc "remove_images", "Remove all app images from servers", hide: true
  def remove_images
    with_lock do
-      on(KAMAL.hosts) do
+      on(KAMAL.app_hosts) do
        execute *KAMAL.auditor.record("Removed all app images"), verbosity: :debug
        execute *KAMAL.app.remove_images
      end
    end
  end

-  desc "remove_app_directory", "Remove the service directory from servers", hide: true
-  def remove_app_directory
+  desc "remove_app_directories", "Remove the app directories from servers", hide: true
+  def remove_app_directories
    with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
        roles = KAMAL.roles_on(host)

        roles.each do |role|
-          execute *KAMAL.auditor.record("Removed #{KAMAL.config.app_directory} on all servers", role: role), verbosity: :debug
+          execute *KAMAL.auditor.record("Removed #{KAMAL.config.app_directory}", role: role), verbosity: :debug
          execute *KAMAL.server.remove_app_directory, raise_on_non_zero_exit: false
        end
+
+        execute *KAMAL.auditor.record("Removed #{KAMAL.config.app_directory}"), verbosity: :debug
+        execute *KAMAL.app.remove_proxy_app_directory, raise_on_non_zero_exit: false
      end
    end
  end

  desc "version", "Show app version currently running on servers"
  def version
-    on(KAMAL.hosts) do |host|
+    on(KAMAL.app_hosts) do |host|
      role = KAMAL.roles_on(host).first
      puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).current_running_version).strip
    end
@@ -350,6 +394,6 @@ class Kamal::Cli::App < Kamal::Cli::Base
    end

    def host_boot_groups
-      KAMAL.config.boot.limit ? KAMAL.hosts.each_slice(KAMAL.config.boot.limit).to_a : [ KAMAL.hosts ]
+      KAMAL.config.boot.limit ? KAMAL.app_hosts.each_slice(KAMAL.config.boot.limit).to_a : [ KAMAL.app_hosts ]
    end
 end
--- a/lib/kamal/cli/app/prepare_assets.rb
+++ b/lib/kamal/cli/app/prepare_assets.rb
@@ -1,4 +1,4 @@
-class Kamal::Cli::App::PrepareAssets
+class Kamal::Cli::App::Assets
  attr_reader :host, :role, :sshkit
  delegate :execute, :capture_with_info, :info, to: :sshkit
  delegate :assets?, to: :role
--- a/lib/kamal/cli/app/boot.rb
+++ b/lib/kamal/cli/app/boot.rb
@@ -70,6 +70,7 @@ class Kamal::Cli::App::Boot
    def stop_old_version(version)
      execute *app.stop(version: version), raise_on_non_zero_exit: false
      execute *app.clean_up_assets if assets?
+      execute *app.clean_up_error_pages if KAMAL.config.error_pages_path
    end

    def release_barrier
--- a/lib/kamal/cli/app/error_pages.rb
+++ b/lib/kamal/cli/app/error_pages.rb
@@ -0,0 +1,33 @@
+class Kamal::Cli::App::ErrorPages
+  ERROR_PAGES_GLOB = "{4??.html,5??.html}"
+
+  attr_reader :host, :sshkit
+  delegate :upload!, :execute, to: :sshkit
+
+  def initialize(host, sshkit)
+    @host = host
+    @sshkit = sshkit
+  end
+
+  def run
+    if KAMAL.config.error_pages_path
+      with_error_pages_tmpdir do |local_error_pages_dir|
+        execute *KAMAL.app.create_error_pages_directory
+        upload! local_error_pages_dir, KAMAL.config.proxy_boot.error_pages_directory, mode: "0700", recursive: true
+      end
+    end
+  end
+
+  private
+    def with_error_pages_tmpdir
+      Dir.mktmpdir("kamal-error-pages") do |tmpdir|
+        error_pages_dir = File.join(tmpdir, KAMAL.config.version)
+        FileUtils.mkdir(error_pages_dir)
+
+        if (files = Dir[File.join(KAMAL.config.error_pages_path, ERROR_PAGES_GLOB)]).any?
+          FileUtils.cp(files, error_pages_dir)
+          yield error_pages_dir
+        end
+      end
+    end
+end
--- a/lib/kamal/cli/base.rb
+++ b/lib/kamal/cli/base.rb
@@ -133,7 +133,13 @@ module Kamal::Cli

      def run_hook(hook, **extra_details)
        if !options[:skip_hooks] && KAMAL.hook.hook_exists?(hook)
-          details = { hosts: KAMAL.hosts.join(","), command: command, subcommand: subcommand }
+          details = {
+            hosts: KAMAL.hosts.join(","),
+            roles: KAMAL.specific_roles&.join(","),
+            lock: KAMAL.holding_lock?.to_s,
+            command: command,
+            subcommand: subcommand
+          }.compact

          say "Running the #{hook} hook...", :magenta
          with_env KAMAL.hook.env(**details, **extra_details) do
@@ -147,12 +153,16 @@ module Kamal::Cli
      end

      def on(*args, &block)
+        pre_connect_if_required
+
+        super
+      end
+
+      def pre_connect_if_required
        if !KAMAL.connected?
          run_hook "pre-connect"
          KAMAL.connected = true
        end
-
-        super
      end

      def command
--- a/lib/kamal/cli/build.rb
+++ b/lib/kamal/cli/build.rb
@@ -14,7 +14,13 @@ class Kamal::Cli::Build < Kamal::Cli::Base
  def push
    cli = self

+    # Ensure pre-connect hooks run before the build, they may needed for a remote builder
+    # or the pre-build hooks.
+    pre_connect_if_required
+
    ensure_docker_installed
+    login_to_registry_locally
+
    run_hook "pre-build"

    uncommitted_changes = Kamal::Git.uncommitted_changes
@@ -61,14 +67,16 @@ class Kamal::Cli::Build < Kamal::Cli::Base

  desc "pull", "Pull app image from registry onto servers"
  def pull
+    login_to_registry_remotely
+
    if (first_hosts = mirror_hosts).any?
      #  Pull on a single host per mirror first to seed them
      say "Pulling image on #{first_hosts.join(", ")} to seed the #{"mirror".pluralize(first_hosts.count)}...", :magenta
      pull_on_hosts(first_hosts)
      say "Pulling image on remaining hosts...", :magenta
-      pull_on_hosts(KAMAL.hosts - first_hosts)
+      pull_on_hosts(KAMAL.app_hosts - first_hosts)
    else
-      pull_on_hosts(KAMAL.hosts)
+      pull_on_hosts(KAMAL.app_hosts)
    end
  end

@@ -159,9 +167,9 @@ class Kamal::Cli::Build < Kamal::Cli::Base
    end

    def mirror_hosts
-      if KAMAL.hosts.many?
+      if KAMAL.app_hosts.many?
        mirror_hosts = Concurrent::Hash.new
-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
          first_mirror = capture_with_info(*KAMAL.builder.first_mirror).strip.presence
          mirror_hosts[first_mirror] ||= host.to_s if first_mirror
        rescue SSHKit::Command::Failed => e
@@ -181,4 +189,16 @@ class Kamal::Cli::Build < Kamal::Cli::Base
        execute *KAMAL.builder.validate_image
      end
    end
+
+    def login_to_registry_locally
+      run_locally do
+        execute *KAMAL.registry.login
+      end
+    end
+
+    def login_to_registry_remotely
+      on(KAMAL.app_hosts) do
+        execute *KAMAL.registry.login
+      end
+    end
 end
--- a/lib/kamal/cli/main.rb
+++ b/lib/kamal/cli/main.rb
@@ -20,9 +20,6 @@ class Kamal::Cli::Main < Kamal::Cli::Base
    runtime = print_runtime do
      invoke_options = deploy_options

-      say "Log into image registry...", :magenta
-      invoke "kamal:cli:registry:login", [], invoke_options.merge(skip_local: options[:skip_push])
-
      if options[:skip_push]
        say "Pull app image...", :magenta
        invoke "kamal:cli:build:pull", [], invoke_options
@@ -52,7 +49,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
    run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s
  end

-  desc "redeploy", "Deploy app to servers without bootstrapping servers, starting kamal-proxy, pruning, and registry login"
+  desc "redeploy", "Deploy app to servers without bootstrapping servers, starting kamal-proxy and pruning"
  option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
  def redeploy
    runtime = print_runtime do
@@ -197,10 +194,10 @@ class Kamal::Cli::Main < Kamal::Cli::Base
    confirming "This will replace Traefik with kamal-proxy and restart all accessories" do
      with_lock do
        if options[:rolling]
-          (KAMAL.hosts | KAMAL.accessory_hosts).each do |host|
+          KAMAL.hosts.each do |host|
            KAMAL.with_specific_hosts(host) do
              say "Upgrading #{host}...", :magenta
-              if KAMAL.hosts.include?(host)
+              if KAMAL.app_hosts.include?(host)
                invoke "kamal:cli:proxy:upgrade", [], options.merge(confirmed: true, rolling: false)
                reset_invocation(Kamal::Cli::Proxy)
              end
@@ -256,7 +253,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
  private
    def container_available?(version)
      begin
-        on(KAMAL.hosts) do
+        on(KAMAL.app_hosts) do
          KAMAL.roles_on(host).each do |role|
            container_id = capture_with_info(*KAMAL.app(role: role, host: host).container_id_for_version(version))
            raise "Container not found" unless container_id.present?
--- a/lib/kamal/cli/proxy.rb
+++ b/lib/kamal/cli/proxy.rb
@@ -13,9 +13,10 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base

        version = capture_with_info(*KAMAL.proxy.version).strip.presence

-        if version && Kamal::Utils.older_version?(version, Kamal::Configuration::PROXY_MINIMUM_VERSION)
-          raise "kamal-proxy version #{version} is too old, run `kamal proxy reboot` in order to update to at least #{Kamal::Configuration::PROXY_MINIMUM_VERSION}"
+        if version && Kamal::Utils.older_version?(version, Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION)
+          raise "kamal-proxy version #{version} is too old, run `kamal proxy reboot` in order to update to at least #{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}"
        end
+        execute *KAMAL.proxy.ensure_apps_config_directory
        execute *KAMAL.proxy.start_or_run
      end
    end
@@ -24,30 +25,75 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
  desc "boot_config <set|get|reset>", "Manage kamal-proxy boot configuration"
  option :publish, type: :boolean, default: true, desc: "Publish the proxy ports on the host"
  option :publish_host_ip, type: :string, repeatable: true, default: nil, desc: "Host IP address to bind HTTP/HTTPS traffic to. Defaults to all interfaces"
-  option :http_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTP_PORT, desc: "HTTP port to publish on the host"
-  option :https_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTPS_PORT, desc: "HTTPS port to publish on the host"
-  option :log_max_size, type: :string, default: Kamal::Configuration::PROXY_LOG_MAX_SIZE, desc: "Max size of proxy logs"
+  option :http_port, type: :numeric, default: Kamal::Configuration::Proxy::Boot::DEFAULT_HTTP_PORT, desc: "HTTP port to publish on the host"
+  option :https_port, type: :numeric, default: Kamal::Configuration::Proxy::Boot::DEFAULT_HTTPS_PORT, desc: "HTTPS port to publish on the host"
+  option :log_max_size, type: :string, default: Kamal::Configuration::Proxy::Boot::DEFAULT_LOG_MAX_SIZE, desc: "Max size of proxy logs"
+  option :registry, type: :string, default: nil, desc: "Registry to use for the proxy image"
+  option :repository, type: :string, default: nil, desc: "Repository for the proxy image"
+  option :image_version, type: :string, default: nil, desc: "Version of the proxy to run"
+  option :metrics_port, type: :numeric, default: nil, desc: "Port to report prometheus metrics on"
+  option :debug, type: :boolean, default: false, desc: "Whether to run the proxy in debug mode"
  option :docker_options, type: :array, default: [], desc: "Docker options to pass to the proxy container", banner: "option=value option2=value2"
  def boot_config(subcommand)
+    proxy_boot_config = KAMAL.config.proxy_boot
+
    case subcommand
    when "set"
      boot_options = [
-        *(KAMAL.config.proxy_publish_args(options[:http_port], options[:https_port], options[:publish_host_ip]) if options[:publish]),
-        *(KAMAL.config.proxy_logging_args(options[:log_max_size])),
+        *(proxy_boot_config.publish_args(options[:http_port], options[:https_port], options[:publish_host_ip]) if options[:publish]),
+        *(proxy_boot_config.logging_args(options[:log_max_size])),
+        *("--expose=#{options[:metrics_port]}" if options[:metrics_port]),
        *options[:docker_options].map { |option| "--#{option}" }
      ]

+      image = [
+        options[:registry].presence,
+        options[:repository].presence || proxy_boot_config.repository_name,
+        proxy_boot_config.image_name
+      ].compact.join("/")
+
+      image_version = options[:image_version]
+
+      run_command_options = { debug: options[:debug] || nil, "metrics-port": options[:metrics_port] }.compact
+      run_command = "kamal-proxy run #{Kamal::Utils.optionize(run_command_options).join(" ")}" if run_command_options.any?
+
      on(KAMAL.proxy_hosts) do |host|
        execute(*KAMAL.proxy.ensure_proxy_directory)
-        upload! StringIO.new(boot_options.join(" ")), KAMAL.config.proxy_options_file
+        if boot_options != proxy_boot_config.default_boot_options
+          upload! StringIO.new(boot_options.join(" ")), proxy_boot_config.options_file
+        else
+          execute *KAMAL.proxy.reset_boot_options, raise_on_non_zero_exit: false
+        end
+
+        if image != proxy_boot_config.image_default
+          upload! StringIO.new(image), proxy_boot_config.image_file
+        else
+          execute *KAMAL.proxy.reset_image, raise_on_non_zero_exit: false
+        end
+
+        if image_version
+          upload! StringIO.new(image_version), proxy_boot_config.image_version_file
+        else
+          execute *KAMAL.proxy.reset_image_version, raise_on_non_zero_exit: false
+        end
+
+        if run_command
+          upload! StringIO.new(run_command), proxy_boot_config.run_command_file
+        else
+          execute *KAMAL.proxy.reset_run_command, raise_on_non_zero_exit: false
+        end
      end
    when "get"
+
      on(KAMAL.proxy_hosts) do |host|
-        puts "Host #{host}: #{capture_with_info(*KAMAL.proxy.get_boot_options)}"
+        puts "Host #{host}: #{capture_with_info(*KAMAL.proxy.boot_config)}"
      end
    when "reset"
      on(KAMAL.proxy_hosts) do |host|
-        execute *KAMAL.proxy.reset_boot_options
+        execute *KAMAL.proxy.reset_boot_options, raise_on_non_zero_exit: false
+        execute *KAMAL.proxy.reset_image, raise_on_non_zero_exit: false
+        execute *KAMAL.proxy.reset_image_version, raise_on_non_zero_exit: false
+        execute *KAMAL.proxy.reset_run_command, raise_on_non_zero_exit: false
      end
    else
      raise ArgumentError, "Unknown boot_config subcommand #{subcommand}"
@@ -71,20 +117,9 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
            "Stopping and removing kamal-proxy on #{host}, if running..."
            execute *KAMAL.proxy.stop, raise_on_non_zero_exit: false
            execute *KAMAL.proxy.remove_container
+            execute *KAMAL.proxy.ensure_apps_config_directory

            execute *KAMAL.proxy.run
-
-            KAMAL.roles_on(host).select(&:running_proxy?).each do |role|
-              app = KAMAL.app(role: role, host: host)
-
-              version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-              endpoint = capture_with_info(*app.container_id_for_version(version)).strip
-
-              if endpoint.present?
-                info "Deploying #{endpoint} for role `#{role}` on #{host}..."
-                execute *app.deploy(target: endpoint)
-              end
-            end
          end
          run_hook "post-proxy-reboot", hosts: host_list
        end
--- a/lib/kamal/cli/server.rb
+++ b/lib/kamal/cli/server.rb
@@ -2,8 +2,10 @@ class Kamal::Cli::Server < Kamal::Cli::Base
  desc "exec", "Run a custom command on the server (use --help to show options)"
  option :interactive, type: :boolean, aliases: "-i", default: false, desc: "Run the command interactively (use for console/bash)"
  def exec(*cmd)
+    pre_connect_if_required
+
    cmd = Kamal::Utils.join_commands(cmd)
-    hosts = KAMAL.hosts | KAMAL.accessory_hosts
+    hosts = KAMAL.hosts

    case
    when options[:interactive]
@@ -27,7 +29,7 @@ class Kamal::Cli::Server < Kamal::Cli::Base
    with_lock do
      missing = []

-      on(KAMAL.hosts | KAMAL.accessory_hosts) do |host|
+      on(KAMAL.hosts) do |host|
        unless execute(*KAMAL.docker.installed?, raise_on_non_zero_exit: false)
          if execute(*KAMAL.docker.superuser?, raise_on_non_zero_exit: false)
            info "Missing Docker on #{host}. Installing…"
--- a/lib/kamal/cli/templates/sample_hooks/post-deploy.sample
+++ b/lib/kamal/cli/templates/sample_hooks/post-deploy.sample
@@ -7,7 +7,7 @@
 # KAMAL_PERFORMER
 # KAMAL_VERSION
 # KAMAL_HOSTS
-# KAMAL_ROLE (if set)
+# KAMAL_ROLES (if set)
 # KAMAL_DESTINATION (if set)
 # KAMAL_RUNTIME

--- a/lib/kamal/cli/templates/sample_hooks/pre-build.sample
+++ b/lib/kamal/cli/templates/sample_hooks/pre-build.sample
@@ -13,7 +13,7 @@
 # KAMAL_PERFORMER
 # KAMAL_VERSION
 # KAMAL_HOSTS
-# KAMAL_ROLE (if set)
+# KAMAL_ROLES (if set)
 # KAMAL_DESTINATION (if set)

 if [ -n "$(git status --porcelain)" ]; then
--- a/lib/kamal/cli/templates/sample_hooks/pre-connect.sample
+++ b/lib/kamal/cli/templates/sample_hooks/pre-connect.sample
@@ -9,7 +9,7 @@
 # KAMAL_PERFORMER
 # KAMAL_VERSION
 # KAMAL_HOSTS
-# KAMAL_ROLE (if set)
+# KAMAL_ROLES (if set)
 # KAMAL_DESTINATION (if set)
 # KAMAL_RUNTIME

--- a/lib/kamal/cli/templates/sample_hooks/pre-deploy.sample
+++ b/lib/kamal/cli/templates/sample_hooks/pre-deploy.sample
@@ -13,7 +13,7 @@
 # KAMAL_HOSTS
 # KAMAL_COMMAND
 # KAMAL_SUBCOMMAND
-# KAMAL_ROLE (if set)
+# KAMAL_ROLES (if set)
 # KAMAL_DESTINATION (if set)

 # Only check the build status for production deployments
@@ -82,11 +82,12 @@ end

 $stdout.sync = true

-puts "Checking build status..."
-attempts = 0
-checks = GithubStatusChecks.new
-
 begin
+  puts "Checking build status..."
+
+  attempts = 0
+  checks = GithubStatusChecks.new
+
  loop do
    case checks.state
    when "success"
--- a/lib/kamal/commander.rb
+++ b/lib/kamal/commander.rb
@@ -5,7 +5,7 @@ require "active_support/core_ext/object/blank"
 class Kamal::Commander
  attr_accessor :verbosity, :holding_lock, :connected
  attr_reader :specific_roles, :specific_hosts
-  delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :proxy_hosts, :accessory_hosts, to: :specifics
+  delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :app_hosts, :proxy_hosts, :accessory_hosts, to: :specifics

  def initialize
    reset
@@ -13,7 +13,7 @@ class Kamal::Commander

  def reset
    self.verbosity = :info
-    self.holding_lock = false
+    self.holding_lock = ENV["KAMAL_LOCK"] == "true"
    self.connected = false
    @specifics = @specific_roles = @specific_hosts = nil
    @config = @config_kwargs = nil
--- a/lib/kamal/commander/specifics.rb
+++ b/lib/kamal/commander/specifics.rb
@@ -11,13 +11,17 @@ class Kamal::Commander::Specifics
    @primary_role = primary_or_first_role(roles_on(primary_host))

    stable_sort!(roles) { |role| role == primary_role ? 0 : 1 }
-    stable_sort!(hosts) { |host| roles_on(host).any? { |role| role == primary_role } ? 0 : 1 }
+    sort_primary_role_hosts_first!(hosts)
  end

  def roles_on(host)
    roles.select { |role| role.hosts.include?(host.to_s) }
  end

+  def app_hosts
+    @app_hosts ||= sort_primary_role_hosts_first!(config.app_hosts & specified_hosts)
+  end
+
  def proxy_hosts
    config.proxy_hosts & specified_hosts
  end
@@ -51,4 +55,8 @@ class Kamal::Commander::Specifics
        specified_hosts
      end
    end
+
+    def sort_primary_role_hosts_first!(hosts)
+      stable_sort!(hosts) { |host| roles_on(host).any? { |role| role == primary_role } ? 0 : 1 }
+    end
 end
--- a/lib/kamal/commands/accessory.rb
+++ b/lib/kamal/commands/accessory.rb
@@ -6,7 +6,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
           :network_args, :publish_args, :env_args, :volume_args, :label_args, :option_args,
           :secrets_io, :secrets_path, :env_directory, :proxy, :running_proxy?, :registry,
           to: :accessory_config
-  delegate :proxy_container_name, to: :config

  def initialize(config, name:)
    super(config)
@@ -37,8 +36,8 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
    docker :container, :stop, service_name
  end

-  def info
-    docker :ps, *service_filter
+  def info(all: false, quiet: false)
+    docker :ps, *("-a" if all), *("-q" if quiet), *service_filter
  end

  def logs(timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil)
--- a/lib/kamal/commands/accessory/proxy.rb
+++ b/lib/kamal/commands/accessory/proxy.rb
@@ -1,5 +1,5 @@
 module Kamal::Commands::Accessory::Proxy
-  delegate :proxy_container_name, to: :config
+  delegate :container_name, to: :"config.proxy_boot", prefix: :proxy

  def deploy(target:)
    proxy_exec :deploy, service_name, *proxy.deploy_command_args(target: target)
--- a/lib/kamal/commands/app.rb
+++ b/lib/kamal/commands/app.rb
@@ -1,5 +1,5 @@
 class Kamal::Commands::App < Kamal::Commands::Base
-  include Assets, Containers, Execution, Images, Logging, Proxy
+  include Assets, Containers, ErrorPages, Execution, Images, Logging, Proxy

  ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]

--- a/lib/kamal/commands/app/error_pages.rb
+++ b/lib/kamal/commands/app/error_pages.rb
@@ -0,0 +1,9 @@
+module Kamal::Commands::App::ErrorPages
+  def create_error_pages_directory
+    make_directory(config.proxy_boot.error_pages_directory)
+  end
+
+  def clean_up_error_pages
+    [ :find, config.proxy_boot.error_pages_directory, "-mindepth", "1", "-maxdepth", "1", "!", "-name", KAMAL.config.version, "-exec", "rm", "-rf", "{} +" ]
+  end
+end
--- a/lib/kamal/commands/app/proxy.rb
+++ b/lib/kamal/commands/app/proxy.rb
@@ -1,5 +1,5 @@
 module Kamal::Commands::App::Proxy
-  delegate :proxy_container_name, to: :config
+  delegate :container_name, to: :"config.proxy_boot", prefix: :proxy

  def deploy(target:)
    proxy_exec :deploy, role.container_prefix, *role.proxy.deploy_command_args(target: target)
@@ -9,6 +9,18 @@ module Kamal::Commands::App::Proxy
    proxy_exec :remove, role.container_prefix
  end

+  def live
+    proxy_exec :resume, role.container_prefix
+  end
+
+  def maintenance(**options)
+    proxy_exec :stop, role.container_prefix, *role.proxy.stop_command_args(**options)
+  end
+
+  def remove_proxy_app_directory
+    remove_directory config.proxy_boot.app_directory
+  end
+
  private
    def proxy_exec(*command)
      docker :exec, proxy_container_name, "kamal-proxy", *command
--- a/lib/kamal/commands/auditor.rb
+++ b/lib/kamal/commands/auditor.rb
@@ -1,5 +1,6 @@
 class Kamal::Commands::Auditor < Kamal::Commands::Base
  attr_reader :details
+  delegate :escape_shell_value, to: Kamal::Utils

  def initialize(config, **details)
    super(config)
@@ -9,11 +10,8 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
  # Runs remotely
  def record(line, **details)
    combine \
-      [ :mkdir, "-p", config.run_directory ],
-      append(
-        [ :echo, audit_tags(**details).except(:version, :service_version, :service).to_s, line ],
-        audit_log_file
-      )
+      make_run_directory,
+      append([ :echo, escape_shell_value(audit_line(line, **details)) ], audit_log_file)
  end

  def reveal
@@ -30,4 +28,12 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
    def audit_tags(**details)
      tags(**self.details, **details)
    end
+
+    def make_run_directory
+      [ :mkdir, "-p", config.run_directory ]
+    end
+
+    def audit_line(line, **details)
+      "#{audit_tags(**details).except(:version, :service_version, :service)} #{line}"
+    end
 end
--- a/lib/kamal/commands/base.rb
+++ b/lib/kamal/commands/base.rb
@@ -68,6 +68,10 @@ module Kamal::Commands
        combine *commands, by: "||"
      end

+      def substitute(*commands)
+        "\$\(#{commands.join(" ")}\)"
+      end
+
      def xargs(command)
        [ :xargs, command ].flatten
      end
--- a/lib/kamal/commands/builder/base.rb
+++ b/lib/kamal/commands/builder/base.rb
@@ -20,7 +20,8 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
      *([ "--builder", builder_name ] unless docker_driver?),
      *build_tag_options(tag_as_dirty: tag_as_dirty),
      *build_options,
-      build_context
+      build_context,
+      "2>&1"
  end

  def pull
--- a/lib/kamal/commands/proxy.rb
+++ b/lib/kamal/commands/proxy.rb
@@ -2,14 +2,7 @@ class Kamal::Commands::Proxy < Kamal::Commands::Base
  delegate :argumentize, :optionize, to: Kamal::Utils

  def run
-    docker :run,
-      "--name", container_name,
-      "--network", "kamal",
-      "--detach",
-      "--restart", "unless-stopped",
-      "--volume", "kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy",
-      "\$\(#{get_boot_options.join(" ")}\)",
-      config.proxy_image
+    pipe boot_config, xargs(docker_run)
  end

  def start
@@ -31,7 +24,7 @@ class Kamal::Commands::Proxy < Kamal::Commands::Base
  def version
    pipe \
      docker(:inspect, container_name, "--format '{{.Config.Image}}'"),
-      [ :cut, "-d:", "-f2" ]
+      [ :awk, "-F:", "'{print \$NF}'" ]
  end

  def logs(timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil)
@@ -65,23 +58,70 @@ class Kamal::Commands::Proxy < Kamal::Commands::Base
  end

  def ensure_proxy_directory
-    make_directory config.proxy_directory
+    make_directory config.proxy_boot.host_directory
  end

  def remove_proxy_directory
-    remove_directory config.proxy_directory
+    remove_directory config.proxy_boot.host_directory
  end

-  def get_boot_options
-    combine [ :cat, config.proxy_options_file ], [ :echo, "\"#{config.proxy_options_default.join(" ")}\"" ], by: "||"
+  def ensure_apps_config_directory
+    make_directory config.proxy_boot.apps_directory
+  end
+
+  def boot_config
+    [ :echo, "#{substitute(read_boot_options)} #{substitute(read_image)}:#{substitute(read_image_version)} #{substitute(read_run_command)}" ]
+  end
+
+  def read_boot_options
+    read_file(config.proxy_boot.options_file, default: config.proxy_boot.default_boot_options.join(" "))
+  end
+
+  def read_image
+    read_file(config.proxy_boot.image_file, default: config.proxy_boot.image_default)
+  end
+
+  def read_image_version
+    read_file(config.proxy_boot.image_version_file, default: Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION)
+  end
+
+  def read_run_command
+    read_file(config.proxy_boot.run_command_file)
  end

  def reset_boot_options
-    remove_file config.proxy_options_file
+    remove_file config.proxy_boot.options_file
+  end
+
+  def reset_image
+    remove_file config.proxy_boot.image_file
+  end
+
+  def reset_image_version
+    remove_file config.proxy_boot.image_version_file
+  end
+
+  def reset_run_command
+    remove_file config.proxy_boot.run_command_file
  end

  private
    def container_name
-      config.proxy_container_name
+      config.proxy_boot.container_name
+    end
+
+    def read_file(file, default: nil)
+      combine [ :cat, file, "2>", "/dev/null" ], [ :echo, "\"#{default}\"" ], by: "||"
+    end
+
+    def docker_run
+      docker \
+        :run,
+        "--name", container_name,
+        "--network", "kamal",
+        "--detach",
+        "--restart", "unless-stopped",
+        "--volume", "kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy",
+        *config.proxy_boot.apps_volume.docker_args
    end
 end
--- a/lib/kamal/configuration.rb
+++ b/lib/kamal/configuration.rb
@@ -10,15 +10,10 @@ class Kamal::Configuration
  delegate :argumentize, :optionize, to: Kamal::Utils

  attr_reader :destination, :raw_config, :secrets
-  attr_reader :accessories, :aliases, :boot, :builder, :env, :logging, :proxy, :servers, :ssh, :sshkit, :registry
+  attr_reader :accessories, :aliases, :boot, :builder, :env, :logging, :proxy, :proxy_boot, :servers, :ssh, :sshkit, :registry

  include Validation

-  PROXY_MINIMUM_VERSION = "v0.8.4"
-  PROXY_HTTP_PORT = 80
-  PROXY_HTTPS_PORT = 443
-  PROXY_LOG_MAX_SIZE = "10m"
-
  class << self
    def create_from(config_file:, destination: nil, version: nil)
      ENV["KAMAL_DESTINATION"] = destination
@@ -68,7 +63,8 @@ class Kamal::Configuration
    @env = Env.new(config: @raw_config.env || {}, secrets: secrets)

    @logging = Logging.new(logging_config: @raw_config.logging)
-    @proxy = Proxy.new(config: self, proxy_config: @raw_config.proxy || {})
+    @proxy = Proxy.new(config: self, proxy_config: @raw_config.proxy)
+    @proxy_boot = Proxy::Boot.new(config: self)
    @ssh = Ssh.new(config: self)
    @sshkit = Sshkit.new(config: self)

@@ -105,6 +101,10 @@ class Kamal::Configuration
    raw_config.minimum_version
  end

+  def service_and_destination
+    [ service, destination ].compact.join("-")
+  end
+
  def roles
    servers.roles
  end
@@ -121,6 +121,10 @@ class Kamal::Configuration
    (roles + accessories).flat_map(&:hosts).uniq
  end

+  def app_hosts
+    roles.flat_map(&:hosts).uniq
+  end
+
  def primary_host
    primary_role&.primary_host
  end
@@ -145,8 +149,12 @@ class Kamal::Configuration
    proxy_roles.flat_map(&:name)
  end

+  def proxy_accessories
+    accessories.select(&:running_proxy?)
+  end
+
  def proxy_hosts
-    proxy_roles.flat_map(&:hosts).uniq
+    (proxy_roles.flat_map(&:hosts) + proxy_accessories.flat_map(&:hosts)).uniq
  end

  def repository
@@ -210,7 +218,7 @@ class Kamal::Configuration
  end

  def app_directory
-    File.join apps_directory, [ service, destination ].compact.join("-")
+    File.join apps_directory, service_and_destination
  end

  def env_directory
@@ -229,6 +237,10 @@ class Kamal::Configuration
    raw_config.asset_path
  end

+  def error_pages_path
+    raw_config.error_pages_path
+  end
+
  def env_tags
    @env_tags ||= if (tags = raw_config.env["tags"])
      tags.collect { |name, config| Env::Tag.new(name, config: config, secrets: secrets) }
@@ -241,42 +253,6 @@ class Kamal::Configuration
    env_tags.detect { |t| t.name == name.to_s }
  end

-  def proxy_publish_args(http_port, https_port, bind_ips = nil)
-    ensure_valid_bind_ips(bind_ips)
-
-    (bind_ips || [ nil ]).map do |bind_ip|
-      bind_ip = format_bind_ip(bind_ip)
-      publish_http = [ bind_ip, http_port, PROXY_HTTP_PORT ].compact.join(":")
-      publish_https = [ bind_ip, https_port, PROXY_HTTPS_PORT ].compact.join(":")
-
-      argumentize "--publish", [ publish_http, publish_https ]
-    end.join(" ")
-  end
-
-  def proxy_logging_args(max_size)
-    argumentize "--log-opt", "max-size=#{max_size}" if max_size.present?
-  end
-
-  def proxy_options_default
-    [ *proxy_publish_args(PROXY_HTTP_PORT, PROXY_HTTPS_PORT), *proxy_logging_args(PROXY_LOG_MAX_SIZE) ]
-  end
-
-  def proxy_image
-    "basecamp/kamal-proxy:#{PROXY_MINIMUM_VERSION}"
-  end
-
-  def proxy_container_name
-    "kamal-proxy"
-  end
-
-  def proxy_directory
-    File.join run_directory, "proxy"
-  end
-
-  def proxy_options_file
-    File.join proxy_directory, "options"
-  end
-
  def to_h
    {
      roles: role_names,
@@ -306,22 +282,26 @@ class Kamal::Configuration
    end

    def ensure_required_keys_present
-      %i[ service image registry servers ].each do |key|
+      %i[ service image registry ].each do |key|
        raise Kamal::ConfigurationError, "Missing required configuration for #{key}" unless raw_config[key].present?
      end

-      unless role(primary_role_name).present?
-        raise Kamal::ConfigurationError, "The primary_role #{primary_role_name} isn't defined"
-      end
+      if raw_config.servers.nil?
+        raise Kamal::ConfigurationError, "No servers or accessories specified" unless raw_config.accessories.present?
+      else
+        unless role(primary_role_name).present?
+          raise Kamal::ConfigurationError, "The primary_role #{primary_role_name} isn't defined"
+        end

-      if primary_role.hosts.empty?
-        raise Kamal::ConfigurationError, "No servers specified for the #{primary_role.name} primary_role"
-      end
+        if primary_role.hosts.empty?
+          raise Kamal::ConfigurationError, "No servers specified for the #{primary_role.name} primary_role"
+        end

-      unless allow_empty_roles?
-        roles.each do |role|
-          if role.hosts.empty?
-            raise Kamal::ConfigurationError, "No servers specified for the #{role.name} role. You can ignore this with allow_empty_roles: true"
+        unless allow_empty_roles?
+          roles.each do |role|
+            if role.hosts.empty?
+              raise Kamal::ConfigurationError, "No servers specified for the #{role.name} role. You can ignore this with allow_empty_roles: true"
+            end
          end
        end
      end
@@ -343,15 +323,6 @@ class Kamal::Configuration
      true
    end

-    def ensure_valid_bind_ips(bind_ips)
-      bind_ips.present? && bind_ips.each do |ip|
-        next if ip =~ Resolv::IPv4::Regex || ip =~ Resolv::IPv6::Regex
-        raise ArgumentError, "Invalid publish IP address: #{ip}"
-      end
-
-      true
-    end
-
    def ensure_retain_containers_valid
      raise Kamal::ConfigurationError, "Must retain at least 1 container" if retain_containers < 1

@@ -383,15 +354,6 @@ class Kamal::Configuration
      true
    end

-    def format_bind_ip(ip)
-      # Ensure IPv6 address inside square brackets - e.g. [::1]
-      if ip =~ Resolv::IPv6::Regex && ip !~ /\[.*\]/
-        "[#{ip}]"
-      else
-        ip
-      end
-    end
-
    def role_names
      raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort
    end
--- a/lib/kamal/configuration/accessory.rb
+++ b/lib/kamal/configuration/accessory.rb
@@ -32,7 +32,7 @@ class Kamal::Configuration::Accessory
  end

  def hosts
-    hosts_from_host || hosts_from_hosts || hosts_from_roles
+    hosts_from_host || hosts_from_hosts || hosts_from_roles || hosts_from_tags
  end

  def port
@@ -201,11 +201,31 @@ class Kamal::Configuration::Accessory
    end

    def hosts_from_roles
-      if accessory_config.key?("roles")
+      if accessory_config.key?("role")
+       config.role(accessory_config["role"])&.hosts
+      elsif accessory_config.key?("roles")
        accessory_config["roles"].flat_map { |role| config.role(role)&.hosts }
      end
    end

+    def hosts_from_tags
+      if accessory_config.key?("tag")
+        extract_hosts_from_config_with_tag(accessory_config["tag"])
+      elsif accessory_config.key?("tags")
+        accessory_config["tags"].flat_map { |tag| extract_hosts_from_config_with_tag(tag) }
+      end
+    end
+
+    def extract_hosts_from_config_with_tag(tag)
+      if (servers_with_roles = config.raw_config.servers).is_a?(Hash)
+        servers_with_roles.flat_map do |role, servers_in_role|
+          servers_in_role.filter_map do |host|
+            host.keys.first if host.is_a?(Hash) && host.values.first.include?(tag)
+          end
+        end
+      end
+    end
+
    def network
      accessory_config["network"] || DEFAULT_NETWORK
    end
@@ -213,6 +233,8 @@ class Kamal::Configuration::Accessory
    def ensure_valid_roles
      if accessory_config["roles"] && (missing_roles = accessory_config["roles"] - config.roles.map(&:name)).any?
        raise Kamal::ConfigurationError, "accessories/#{name}: unknown roles #{missing_roles.join(", ")}"
+      elsif accessory_config["role"] && !config.role(accessory_config["role"])
+        raise Kamal::ConfigurationError, "accessories/#{name}: unknown role #{accessory_config["role"]}"
      end
    end
 end
--- a/lib/kamal/configuration/docs/accessory.yml
+++ b/lib/kamal/configuration/docs/accessory.yml
@@ -46,13 +46,18 @@ accessories:

    # Accessory hosts
    #
-    # Specify one of `host`, `hosts`, or `roles`:
+    # Specify one of `host`, `hosts`, `role`, `roles`, `tag` or `tags`:
    host: mysql-db1
    hosts:
      - mysql-db1
      - mysql-db2
+    role: mysql
    roles:
      - mysql
+    tag: writer
+    tags:
+      - writer
+      - reader

    # Custom command
    #
--- a/lib/kamal/configuration/docs/configuration.yml
+++ b/lib/kamal/configuration/docs/configuration.yml
@@ -82,6 +82,12 @@ asset_path: /path/to/assets
 # See https://kamal-deploy.org/docs/hooks for more information:
 hooks_path: /user_home/kamal/hooks

+# Error pages
+#
+# A directory relative to the app root to find error pages for the proxy to serve.
+# Any files in the format 4xx.html or 5xx.html will be copied to the hosts.
+error_pages_path: public
+
 # Require destinations
 #
 # Whether deployments require a destination to be specified, defaults to `false`:
--- a/lib/kamal/configuration/docs/env.yml
+++ b/lib/kamal/configuration/docs/env.yml
@@ -51,6 +51,37 @@ env:
  secret:
    - DB_PASSWORD

+# Aliased secrets
+#
+# You can also alias secrets to other secrets using a `:` separator.
+#
+# This is useful when the ENV name is different from the secret name. For example, if you have two
+# places where you need to define the ENV variable `DB_PASSWORD`, but the value is different depending
+# on the context.
+#
+# ```shell
+# SECRETS=$(kamal secrets fetch ...)
+#
+# MAIN_DB_PASSWORD=$(kamal secrets extract MAIN_DB_PASSWORD $SECRETS)
+# SECONDARY_DB_PASSWORD=$(kamal secrets extract SECONDARY_DB_PASSWORD $SECRETS)
+# ```
+env:
+  secret:
+    - DB_PASSWORD:MAIN_DB_PASSWORD
+  tags:
+    secondary_db:
+      secret:
+        - DB_PASSWORD:SECONDARY_DB_PASSWORD
+accessories:
+  main_db_accessory:
+    env:
+      secret:
+        - DB_PASSWORD:MAIN_DB_PASSWORD
+  secondary_db_accessory:
+    env:
+      secret:
+        - DB_PASSWORD:SECONDARY_DB_PASSWORD
+
 # Tags
 #
 # Tags are used to add extra env variables to specific hosts.
--- a/lib/kamal/configuration/docs/proxy.yml
+++ b/lib/kamal/configuration/docs/proxy.yml
@@ -10,11 +10,6 @@
 # They are application-specific, so they are not shared when multiple applications
 # run on the same proxy.
 #
-# The proxy is enabled by default on the primary role but can be disabled by
-# setting `proxy: false`.
-#
-# It is disabled by default on all other roles but can be enabled by setting
-# `proxy: true` or providing a proxy configuration.
 proxy:

  # Hosts
@@ -52,6 +47,13 @@ proxy:
  # Defaults to `false`:
  ssl: true

+  # SSL redirect
+  #
+  # By default, kamal-proxy will redirect all HTTP requests to HTTPS when SSL is enabled.
+  # If you prefer that HTTP traffic is passed through to your application (along with
+  # HTTPS traffic), you can disable this redirect by setting `ssl_redirect: false`:
+  ssl_redirect: false
+
  # Forward headers
  #
  # Whether to forward the `X-Forwarded-For` and `X-Forwarded-Proto` headers.
@@ -106,3 +108,30 @@ proxy:
    response_headers:
      - X-Request-ID
      - X-Request-Start
+
+# Enabling/disabling the proxy on roles
+#
+# The proxy is enabled by default on the primary role but can be disabled by
+# setting `proxy: false` in the primary role's configuration.
+#
+# ```yaml
+# servers:
+#   web:
+#     hosts:
+#      - ...
+#     proxy: false
+# ```
+#
+# It is disabled by default on all other roles but can be enabled by setting
+# `proxy: true` or providing a proxy configuration for that role.
+#
+# ```yaml
+# servers:
+#   web:
+#     hosts:
+#      - ...
+#   web2:
+#     hosts:
+#      - ...
+#     proxy: true
+# ```
--- a/lib/kamal/configuration/env.rb
+++ b/lib/kamal/configuration/env.rb
@@ -1,8 +1,7 @@
 class Kamal::Configuration::Env
  include Kamal::Configuration::Validation

-  attr_reader :context, :secrets
-  attr_reader :clear, :secret_keys
+  attr_reader :context, :clear, :secret_keys
  delegate :argumentize, to: Kamal::Utils

  def initialize(config:, secrets:, context: "env")
@@ -18,12 +17,22 @@ class Kamal::Configuration::Env
  end

  def secrets_io
-    Kamal::EnvFile.new(secret_keys.to_h { |key| [ key, secrets[key] ] }).to_io
+    Kamal::EnvFile.new(aliased_secrets).to_io
  end

  def merge(other)
    self.class.new \
      config: { "clear" => clear.merge(other.clear), "secret" => secret_keys | other.secret_keys },
-      secrets: secrets
+      secrets: @secrets
  end
+
+  private
+    def aliased_secrets
+      secret_keys.to_h { |key| extract_alias(key) }.transform_values { |secret_key| @secrets[secret_key] }
+    end
+
+    def extract_alias(key)
+      key_name, key_aliased_to = key.split(":", 2)
+      [ key_name, key_aliased_to || key_name ]
+    end
 end
--- a/lib/kamal/configuration/proxy.rb
+++ b/lib/kamal/configuration/proxy.rb
@@ -11,6 +11,7 @@ class Kamal::Configuration::Proxy
  def initialize(config:, proxy_config:, context: "proxy")
    @config = config
    @proxy_config = proxy_config
+    @proxy_config = {} if @proxy_config.nil?
    validate! @proxy_config, with: Kamal::Configuration::Validator::Proxy, context: context
  end

@@ -42,8 +43,10 @@ class Kamal::Configuration::Proxy
      "max-request-body": proxy_config.dig("buffering", "max_request_body"),
      "max-response-body": proxy_config.dig("buffering", "max_response_body"),
      "forward-headers": proxy_config.dig("forward_headers"),
+      "tls-redirect": proxy_config.dig("ssl_redirect"),
      "log-request-header": proxy_config.dig("logging", "request_headers") || DEFAULT_LOG_REQUEST_HEADERS,
-      "log-response-header": proxy_config.dig("logging", "response_headers")
+      "log-response-header": proxy_config.dig("logging", "response_headers"),
+      "error-pages": error_pages
    }.compact
  end

@@ -51,6 +54,17 @@ class Kamal::Configuration::Proxy
    optionize ({ target: "#{target}:#{app_port}" }).merge(deploy_options), with: "="
  end

+  def stop_options(drain_timeout: nil, message: nil)
+    {
+      "drain-timeout": seconds_duration(drain_timeout),
+      message: message
+    }.compact
+  end
+
+  def stop_command_args(**options)
+    optionize stop_options(**options), with: "="
+  end
+
  def merge(other)
    self.class.new config: config, proxy_config: proxy_config.deep_merge(other.proxy_config)
  end
@@ -59,4 +73,8 @@ class Kamal::Configuration::Proxy
    def seconds_duration(value)
      value ? "#{value}s" : nil
    end
+
+    def error_pages
+      File.join config.proxy_boot.error_pages_container_directory, config.version if config.error_pages_path
+    end
 end
--- a/lib/kamal/configuration/proxy/boot.rb
+++ b/lib/kamal/configuration/proxy/boot.rb
@@ -0,0 +1,121 @@
+class Kamal::Configuration::Proxy::Boot
+  MINIMUM_VERSION = "v0.9.0"
+  DEFAULT_HTTP_PORT = 80
+  DEFAULT_HTTPS_PORT = 443
+  DEFAULT_LOG_MAX_SIZE = "10m"
+
+  attr_reader :config
+  delegate :argumentize, :optionize, to: Kamal::Utils
+
+  def initialize(config:)
+    @config = config
+  end
+
+  def publish_args(http_port, https_port, bind_ips = nil)
+    ensure_valid_bind_ips(bind_ips)
+
+    (bind_ips || [ nil ]).map do |bind_ip|
+      bind_ip = format_bind_ip(bind_ip)
+      publish_http = [ bind_ip, http_port, DEFAULT_HTTP_PORT ].compact.join(":")
+      publish_https = [ bind_ip, https_port, DEFAULT_HTTPS_PORT ].compact.join(":")
+
+      argumentize "--publish", [ publish_http, publish_https ]
+    end.join(" ")
+  end
+
+  def logging_args(max_size)
+    argumentize "--log-opt", "max-size=#{max_size}" if max_size.present?
+  end
+
+  def default_boot_options
+    [
+      *(publish_args(DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT, nil)),
+      *(logging_args(DEFAULT_LOG_MAX_SIZE))
+    ]
+  end
+
+  def repository_name
+    "basecamp"
+  end
+
+  def image_name
+    "kamal-proxy"
+  end
+
+  def image_default
+    "#{repository_name}/#{image_name}"
+  end
+
+  def container_name
+    "kamal-proxy"
+  end
+
+  def host_directory
+    File.join config.run_directory, "proxy"
+  end
+
+  def options_file
+    File.join host_directory, "options"
+  end
+
+  def image_file
+    File.join host_directory, "image"
+  end
+
+  def image_version_file
+    File.join host_directory, "image_version"
+  end
+
+  def run_command_file
+    File.join host_directory, "run_command"
+  end
+
+  def apps_directory
+    File.join host_directory, "apps-config"
+  end
+
+  def apps_container_directory
+    "/home/kamal-proxy/.apps-config"
+  end
+
+  def apps_volume
+    Kamal::Configuration::Volume.new \
+      host_path: apps_directory,
+      container_path: apps_container_directory
+  end
+
+  def app_directory
+    File.join apps_directory, config.service_and_destination
+  end
+
+  def app_container_directory
+    File.join apps_container_directory, config.service_and_destination
+  end
+
+  def error_pages_directory
+    File.join app_directory, "error_pages"
+  end
+
+  def error_pages_container_directory
+    File.join app_container_directory, "error_pages"
+  end
+
+  private
+    def ensure_valid_bind_ips(bind_ips)
+      bind_ips.present? && bind_ips.each do |ip|
+        next if ip =~ Resolv::IPv4::Regex || ip =~ Resolv::IPv6::Regex
+        raise ArgumentError, "Invalid publish IP address: #{ip}"
+      end
+
+      true
+    end
+
+    def format_bind_ip(ip)
+      # Ensure IPv6 address inside square brackets - e.g. [::1]
+      if ip =~ Resolv::IPv6::Regex && ip !~ /\A\[.*\]\z/
+        "[#{ip}]"
+      else
+        ip
+      end
+    end
+end
--- a/lib/kamal/configuration/servers.rb
+++ b/lib/kamal/configuration/servers.rb
@@ -13,6 +13,13 @@ class Kamal::Configuration::Servers

  private
    def role_names
-      servers_config.is_a?(Array) ? [ "web" ] : servers_config.keys.sort
+      case servers_config
+      when Array
+        [ "web" ]
+      when NilClass
+        []
+      else
+        servers_config.keys.sort
+      end
    end
 end
--- a/lib/kamal/configuration/validator.rb
+++ b/lib/kamal/configuration/validator.rb
@@ -168,4 +168,10 @@ class Kamal::Configuration::Validator
      unknown_keys.reject! { |key| extension?(key) } if allow_extensions?
      unknown_keys_error unknown_keys if unknown_keys.present?
    end
+
+    def validate_docker_options!(options)
+      if options
+        error "Cannot set restart policy in docker options, unless-stopped is required" if options["restart"]
+      end
+    end
 end
--- a/lib/kamal/configuration/validator/accessory.rb
+++ b/lib/kamal/configuration/validator/accessory.rb
@@ -2,8 +2,10 @@ class Kamal::Configuration::Validator::Accessory < Kamal::Configuration::Validat
  def validate!
    super

-    if (config.keys & [ "host", "hosts", "roles" ]).size != 1
-      error "specify one of `host`, `hosts` or `roles`"
+    if (config.keys & [ "host", "hosts", "role", "roles", "tag", "tags" ]).size != 1
+      error "specify one of `host`, `hosts`, `role`, `roles`, `tag` or `tags`"
    end
+
+    validate_docker_options!(config["options"])
  end
 end
--- a/lib/kamal/configuration/validator/role.rb
+++ b/lib/kamal/configuration/validator/role.rb
@@ -6,6 +6,7 @@ class Kamal::Configuration::Validator::Role < Kamal::Configuration::Validator
      validate_servers!(config)
    else
      super
+      validate_docker_options!(config["options"])
    end
  end
 end
--- a/lib/kamal/configuration/validator/servers.rb
+++ b/lib/kamal/configuration/validator/servers.rb
@@ -1,6 +1,6 @@
 class Kamal::Configuration::Validator::Servers < Kamal::Configuration::Validator
  def validate!
-    validate_type! config, Array, Hash
+    validate_type! config, Array, Hash, NilClass

    validate_servers! config if config.is_a?(Array)
  end
--- a/lib/kamal/secrets/adapters/aws_secrets_manager.rb
+++ b/lib/kamal/secrets/adapters/aws_secrets_manager.rb
@@ -26,6 +26,7 @@ class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Ba
    def get_from_secrets_manager(secrets, account: nil)
      args = [ "aws", "secretsmanager", "batch-get-secret-value", "--secret-id-list" ] + secrets.map(&:shellescape)
      args += [ "--profile", account.shellescape ] if account
+      args += [ "--output", "json" ]
      cmd = args.join(" ")

      `#{cmd}`.tap do |secrets|
--- a/lib/kamal/secrets/dotenv/inline_command_substitution.rb
+++ b/lib/kamal/secrets/dotenv/inline_command_substitution.rb
@@ -4,7 +4,7 @@ class Kamal::Secrets::Dotenv::InlineCommandSubstitution
      ::Dotenv::Parser.substitutions.map! { |sub| sub == ::Dotenv::Substitutions::Command ? self : sub }
    end

-    def call(value, _env, overwrite: false)
+    def call(value, env, overwrite: false)
      # Process interpolated shell commands
      value.gsub(Dotenv::Substitutions::Command.singleton_class::INTERPOLATED_SHELL_COMMAND) do |*|
        # Eliminate opening and closing parentheses
@@ -14,6 +14,7 @@ class Kamal::Secrets::Dotenv::InlineCommandSubstitution
          # Command is escaped, don't replace it.
          $LAST_MATCH_INFO[0][1..]
        else
+          command = ::Dotenv::Substitutions::Variable.call(command, env)
          if command =~ /\A\s*kamal\s*secrets\s+/
            # Inline the command
            inline_secrets_command(command)
--- a/lib/kamal/version.rb
+++ b/lib/kamal/version.rb
@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "2.5.3"
+  VERSION = "2.6.1"
 end
--- a/test/cli/accessory_test.rb
+++ b/test/cli/accessory_test.rb
@@ -115,6 +115,7 @@ class CliAccessoryTest < CliTestCase

  test "exec" do
    run_command("exec", "mysql", "mysql -v").tap do |output|
+      assert_match "docker login private.registry -u [REDACTED] -p [REDACTED]", output
      assert_match "Launching command from new container", output
      assert_match "mysql -v", output
    end
@@ -251,6 +252,19 @@ class CliAccessoryTest < CliTestCase
    end
  end

+  test "boot with web role filter" do
+    run_command("boot", "redis", "-r", "web").tap do |output|
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
+    end
+  end
+
+  test "boot with workers role filter" do
+    run_command("boot", "redis", "-r", "workers").tap do |output|
+      assert_no_match "docker run", output
+    end
+  end
+
  private
    def run_command(*command)
      stdouted { Kamal::Cli::Accessory.start([ *command, "-c", "test/fixtures/deploy_with_accessories_with_different_registries.yml" ]) }
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -192,6 +192,34 @@ class CliAppTest < CliTestCase
    Thread.report_on_exception = true
  end

+  test "boot with only workers" do
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running").at_least_once # workers health check
+
+    run_command("boot", config: :with_only_workers, host: nil).tap do |output|
+      assert_match /First workers container is healthy on 1.1.1.\d, booting any other roles/, output
+      assert_no_match "kamal-proxy", output
+    end
+  end
+
+  test "boot with error pages" do
+    with_error_pages(directory: "public") do
+      stub_running
+      run_command("boot", config: :with_error_pages).tap do |output|
+        assert_match /Uploading .*kamal-error-pages.*\/latest to \.kamal\/proxy\/apps-config\/app\/error_pages/, output
+        assert_match "docker tag dhh/app:latest dhh/app:latest", output
+        assert_match /docker run --detach --restart unless-stopped --name app-web-latest --network kamal --hostname 1.1.1.1-[0-9a-f]{12} /, output
+        assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+        assert_match "Running /usr/bin/env find .kamal/proxy/apps-config/app/error_pages -mindepth 1 -maxdepth 1 ! -name latest -exec rm -rf {} + on 1.1.1.1", output
+      end
+    end
+  end
+
  test "start" do
    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("999") # old version

@@ -244,9 +272,11 @@ class CliAppTest < CliTestCase

  test "remove" do
    run_command("remove").tap do |output|
-      assert_match /#{Regexp.escape("sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop")}/, output
-      assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output
-      assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output
+      assert_match "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop", output
+      assert_match "docker container prune --force --filter label=service=app", output
+      assert_match "docker image prune --all --force --filter label=service=app", output
+      assert_match "rm -r .kamal/apps/app on 1.1.1.1", output
+      assert_match "rm -r .kamal/proxy/apps-config/app on 1.1.1.1", output
    end
  end

@@ -268,12 +298,27 @@ class CliAppTest < CliTestCase
    end
  end

+  test "remove_app_directories" do
+    run_command("remove_app_directories").tap do |output|
+      assert_match "rm -r .kamal/apps/app on 1.1.1.1", output
+      assert_match "rm -r .kamal/proxy/apps-config/app on 1.1.1.1", output
+    end
+  end
+
  test "exec" do
    run_command("exec", "ruby -v").tap do |output|
+      assert_match "docker login -u [REDACTED] -p [REDACTED]", output
      assert_match "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:latest ruby -v", output
    end
  end

+  test "exec without command fails" do
+    error = assert_raises(ArgumentError, "Exec requires a command to be specified") do
+      run_command("exec")
+    end
+    assert_equal "No command provided. You must specify a command to execute.", error.message
+  end
+
  test "exec separate arguments" do
    run_command("exec", "ruby", " -v").tap do |output|
      assert_match "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:latest ruby -v", output
@@ -312,18 +357,25 @@ class CliAppTest < CliTestCase
  end

  test "exec interactive" do
+    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
    SSHKit::Backend::Abstract.any_instance.expects(:exec)
      .with("ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:latest ruby -v'")
+
    run_command("exec", "-i", "ruby -v").tap do |output|
+      assert_hook_ran "pre-connect", output
+      assert_match "docker login -u [REDACTED] -p [REDACTED]", output
      assert_match "Get most recent version available as an image...", output
      assert_match "Launching interactive command with version latest via SSH from new container on 1.1.1.1...", output
    end
  end

  test "exec interactive with reuse" do
+    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
    SSHKit::Backend::Abstract.any_instance.expects(:exec)
      .with("ssh -t root@1.1.1.1 -p 22 'docker exec -it app-web-999 ruby -v'")
+
    run_command("exec", "-i", "--reuse", "ruby -v").tap do |output|
+      assert_hook_ran "pre-connect", output
      assert_match "Get current version of running container...", output
      assert_match "Running /usr/bin/env sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done on 1.1.1.1", output
      assert_match "Launching interactive command with version 999 via SSH from existing container on 1.1.1.1...", output
@@ -437,6 +489,24 @@ class CliAppTest < CliTestCase
    end
  end

+  test "live" do
+    run_command("live").tap do |output|
+      assert_match "docker exec kamal-proxy kamal-proxy resume app-web on 1.1.1.1", output
+    end
+  end
+
+  test "maintenance" do
+    run_command("maintenance").tap do |output|
+      assert_match "docker exec kamal-proxy kamal-proxy stop app-web --drain-timeout=\"30s\" on 1.1.1.1", output
+    end
+  end
+
+  test "maintenance with options" do
+    run_command("maintenance", "--message", "Hello", "--drain_timeout", "10").tap do |output|
+      assert_match "docker exec kamal-proxy kamal-proxy stop app-web --drain-timeout=\"10s\" --message=\"Hello\" on 1.1.1.1", output
+    end
+  end
+
  private
    def run_command(*command, config: :with_accessories, host: "1.1.1.1", allow_execute_error: false)
      stdouted do
--- a/test/cli/build_test.rb
+++ b/test/cli/build_test.rb
@@ -21,11 +21,12 @@ class CliBuildTest < CliTestCase
        .returns("")

      run_command("push", "--verbose").tap do |output|
+        assert_hook_ran "pre-connect", output
        assert_hook_ran "pre-build", output
        assert_match /Cloning repo into build directory/, output
        assert_match /git -C #{Dir.tmpdir}\/kamal-clones\/app-#{pwd_sha} clone #{Dir.pwd}/, output
        assert_match /docker --version && docker buildx version/, output
-        assert_match /docker buildx build --output=type=registry --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
+        assert_match /docker buildx build --output=type=registry --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. 2>&1 as .*@localhost/, output
      end
    end
  end
@@ -47,7 +48,7 @@ class CliBuildTest < CliTestCase
        assert_match /Cloning repo into build directory/, output
        assert_match /git -C #{Dir.tmpdir}\/kamal-clones\/app-#{pwd_sha} clone #{Dir.pwd}/, output
        assert_match /docker --version && docker buildx version/, output
-        assert_match /docker buildx build --output=type=docker --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
+        assert_match /docker buildx build --output=type=docker --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. 2>&1 as .*@localhost/, output
      end
    end
  end
@@ -57,6 +58,7 @@ class CliBuildTest < CliTestCase
      stub_setup

      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with { |*args| args[0..1] == [ :docker, :login ] }

      SSHKit::Backend::Abstract.any_instance.expects(:execute)
        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd, "--recurse-submodules")
@@ -70,7 +72,7 @@ class CliBuildTest < CliTestCase
      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :submodule, :update, "--init")

      SSHKit::Backend::Abstract.any_instance.expects(:execute)
-        .with(:docker, :buildx, :build, "--output=type=registry", "--platform", "linux/amd64", "--builder", "kamal-local-docker-container", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".")
+        .with(:docker, :buildx, :build, "--output=type=registry", "--platform", "linux/amd64", "--builder", "kamal-local-docker-container", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".", "2>&1")

      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
        .with(:git, "-C", anything, :"rev-parse", :HEAD)
@@ -94,7 +96,7 @@ class CliBuildTest < CliTestCase
      assert_no_match /Cloning repo into build directory/, output
      assert_hook_ran "pre-build", output
      assert_match /docker --version && docker buildx version/, output
-      assert_match /docker buildx build --output=type=registry --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile . as .*@localhost/, output
+      assert_match /docker buildx build --output=type=registry --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile . 2>&1 as .*@localhost/, output
    end
  end

@@ -103,6 +105,7 @@ class CliBuildTest < CliTestCase
      stub_setup

      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with { |*args| args[0..1] == [ :docker, :login ] }

      SSHKit::Backend::Abstract.any_instance.expects(:execute)
        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd, "--recurse-submodules")
@@ -139,6 +142,9 @@ class CliBuildTest < CliTestCase
      SSHKit::Backend::Abstract.any_instance.expects(:execute)
        .with(:docker, "--version", "&&", :docker, :buildx, "version")

+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| args[0..1] == [ :docker, :login ] }
+
      SSHKit::Backend::Abstract.any_instance.expects(:execute)
        .with(:docker, :buildx, :rm, "kamal-local-docker-container")

@@ -160,7 +166,7 @@ class CliBuildTest < CliTestCase
        .returns("")

      SSHKit::Backend::Abstract.any_instance.expects(:execute)
-        .with(:docker, :buildx, :build, "--output=type=registry", "--platform", "linux/amd64", "--builder", "kamal-local-docker-container", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".")
+        .with(:docker, :buildx, :build, "--output=type=registry", "--platform", "linux/amd64", "--builder", "kamal-local-docker-container", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".", "2>&1")

      run_command("push").tap do |output|
        assert_match /WARN Missing compatible builder, so creating a new one first/, output
@@ -302,7 +308,7 @@ class CliBuildTest < CliTestCase
      run_command("dev", "--verbose").tap do |output|
        assert_no_match(/Cloning repo into build directory/, output)
        assert_match(/docker --version && docker buildx version/, output)
-        assert_match(/docker buildx build --output=type=docker --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999-dirty -t dhh\/app:latest-dirty --label service="app" --file Dockerfile \. as .*@localhost/, output)
+        assert_match(/docker buildx build --output=type=docker --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999-dirty -t dhh\/app:latest-dirty --label service="app" --file Dockerfile \. 2>&1 as .*@localhost/, output)
      end
    end
  end
@@ -314,14 +320,14 @@ class CliBuildTest < CliTestCase
      run_command("dev", "--output=local", "--verbose").tap do |output|
        assert_no_match(/Cloning repo into build directory/, output)
        assert_match(/docker --version && docker buildx version/, output)
-        assert_match(/docker buildx build --output=type=local --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999-dirty -t dhh\/app:latest-dirty --label service="app" --file Dockerfile \. as .*@localhost/, output)
+        assert_match(/docker buildx build --output=type=local --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999-dirty -t dhh\/app:latest-dirty --label service="app" --file Dockerfile \. 2>&1 as .*@localhost/, output)
      end
    end
  end

  private
    def run_command(*command, fixture: :with_accessories)
-      stdouted { Kamal::Cli::Build.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) }
+      stdouted { stderred { Kamal::Cli::Build.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) } }
    end

    def stub_dependency_checks
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -21,7 +21,6 @@ class CliMainTest < CliTestCase
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
    # deploy
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: true))
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -32,7 +31,6 @@ class CliMainTest < CliTestCase
      assert_match /Ensure Docker is installed.../, output
      # deploy
      assert_match /Acquiring the deploy lock/, output
-      assert_match /Log into image registry/, output
      assert_match /Pull app image/, output
      assert_match /Ensure kamal-proxy is running/, output
      assert_match /Detect stale containers/, output
@@ -45,7 +43,6 @@ class CliMainTest < CliTestCase
    with_test_secrets("secrets" => "DB_PASSWORD=secret") do
      invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true }

-      Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
      Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
      Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
      Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -56,7 +53,6 @@ class CliMainTest < CliTestCase

      run_command("deploy", "--verbose").tap do |output|
        assert_hook_ran "pre-connect", output
-        assert_match /Log into image registry/, output
        assert_match /Build and push app image/, output
        assert_hook_ran "pre-deploy", output
        assert_match /Ensure kamal-proxy is running/, output
@@ -70,7 +66,6 @@ class CliMainTest < CliTestCase
  test "deploy with skip_push" do
    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }

-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: true))
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -79,7 +74,6 @@ class CliMainTest < CliTestCase

    run_command("deploy", "--skip_push").tap do |output|
      assert_match /Acquiring the deploy lock/, output
-      assert_match /Log into image registry/, output
      assert_match /Pull app image/, output
      assert_match /Ensure kamal-proxy is running/, output
      assert_match /Detect stale containers/, output
@@ -122,6 +116,32 @@ class CliMainTest < CliTestCase
    end
  end

+  test "deploy when inheriting lock" do
+    Thread.report_on_exception = false
+
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
+
+    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+
+    with_kamal_lock_env do
+      KAMAL.reset
+      run_command("deploy").tap do |output|
+        assert_no_match /Acquiring the deploy lock/, output
+        assert_match /Build and push app image/, output
+        assert_match /Ensure kamal-proxy is running/, output
+        assert_match /Detect stale containers/, output
+        assert_match /Prune old containers and images/, output
+        assert_no_match /Releasing the deploy lock/, output
+      end
+    end
+  end
+
  test "deploy error when locking" do
    Thread.report_on_exception = false

@@ -153,11 +173,11 @@ class CliMainTest < CliTestCase
    end
  end

-  test "deploy errors during outside section leave remove lock" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, :skip_local => false }
+  test "deploy errors during outside section leave remote lock" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }

    Kamal::Cli::Main.any_instance.expects(:invoke)
-      .with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
+      .with("kamal:cli:build:deliver", [], invoke_options)
      .raises(RuntimeError)

    assert_not KAMAL.holding_lock?
@@ -170,7 +190,6 @@ class CliMainTest < CliTestCase
  test "deploy with skipped hooks" do
    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => true }

-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -185,7 +204,6 @@ class CliMainTest < CliTestCase
  test "deploy with missing secrets" do
    invoke_options = { "config_file" => "test/fixtures/deploy_with_secrets.yml", "version" => "999", "skip_hooks" => false }

-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -570,4 +588,11 @@ class CliMainTest < CliTestCase
    def assert_file(file, content)
      assert_match content, File.read(file)
    end
+
+    def with_kamal_lock_env
+      ENV["KAMAL_LOCK"] = "true"
+      yield
+    ensure
+      ENV.delete("KAMAL_LOCK")
+    end
 end
--- a/test/cli/proxy_test.rb
+++ b/test/cli/proxy_test.rb
@@ -4,25 +4,26 @@ class CliProxyTest < CliTestCase
  test "boot" do
    run_command("boot").tap do |output|
      assert_match "docker login", output
-      assert_match "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image}", output
+      assert_match "mkdir -p .kamal/proxy/apps-config", output
+      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
    end
  end

  test "boot old version" do
    Thread.report_on_exception = false
    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2")
+      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :awk, "-F:", "'{print $NF}'")
      .returns("v0.0.1")
      .at_least_once

    exception = assert_raises do
      run_command("boot").tap do |output|
        assert_match "docker login", output
-        assert_match "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image}", output
+        assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
      end
    end

-    assert_includes exception.message, "kamal-proxy version v0.0.1 is too old, run `kamal proxy reboot` in order to update to at least #{Kamal::Configuration::PROXY_MINIMUM_VERSION}"
+    assert_includes exception.message, "kamal-proxy version v0.0.1 is too old, run `kamal proxy reboot` in order to update to at least #{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}"
  ensure
    Thread.report_on_exception = false
  end
@@ -30,53 +31,33 @@ class CliProxyTest < CliTestCase
  test "boot correct version" do
    Thread.report_on_exception = false
    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2")
-      .returns(Kamal::Configuration::PROXY_MINIMUM_VERSION)
+      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :awk, "-F:", "'{print $NF}'")
+      .returns(Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION)
      .at_least_once

    run_command("boot").tap do |output|
      assert_match "docker login", output
-      assert_match "docker container start kamal-proxy || docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image}", output
+      assert_match "docker container start kamal-proxy || echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
    end
  ensure
    Thread.report_on_exception = false
  end

  test "reboot" do
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet")
-      .returns("abcdefabcdef")
-      .at_least_once
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with { |*args| args[0..1] == [ :sh, "-c" ] }
-      .returns("123")
-      .at_least_once
-
    run_command("reboot", "-y").tap do |output|
      assert_match "docker container stop kamal-proxy on 1.1.1.1", output
      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.1", output
-      assert_match "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image} on 1.1.1.1", output
-      assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target=\"abcdefabcdef:80\" --deploy-timeout=\"6s\" --drain-timeout=\"30s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\" on 1.1.1.1", output
+      assert_match "mkdir -p .kamal/proxy/apps-config on 1.1.1.1", output
+      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config on 1.1.1.1", output

      assert_match "docker container stop kamal-proxy on 1.1.1.2", output
      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.2", output
-      assert_match "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image} on 1.1.1.2", output
-      assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target=\"abcdefabcdef:80\" --deploy-timeout=\"6s\" --drain-timeout=\"30s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\" on 1.1.1.2", output
+      assert_match "mkdir -p .kamal/proxy/apps-config on 1.1.1.1", output
+      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config on 1.1.1.2", output
    end
  end

  test "reboot --rolling" do
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet")
-      .returns("abcdefabcdef")
-      .at_least_once
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with { |*args| args[0..1] == [ :sh, "-c" ] }
-      .returns("123")
-      .at_least_once
-
    run_command("reboot", "--rolling", "-y").tap do |output|
      assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.1", output
    end
@@ -179,8 +160,8 @@ class CliProxyTest < CliTestCase
    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("12345678")

    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2")
-      .returns(Kamal::Configuration::PROXY_MINIMUM_VERSION)
+      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :awk, "-F:", "'{print $NF}'")
+      .returns(Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION)

    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
@@ -196,7 +177,7 @@ class CliProxyTest < CliTestCase
      assert_match "/usr/bin/env mkdir -p .kamal", output
      assert_match "docker network create kamal", output
      assert_match "docker login -u [REDACTED] -p [REDACTED]", output
-      assert_match "docker container start kamal-proxy || docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION}", output
+      assert_match "docker container start kamal-proxy || echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
      assert_match "/usr/bin/env mkdir -p .kamal", output
      assert_match %r{docker rename app-web-latest app-web-latest_replaced_.*}, output
      assert_match "/usr/bin/env mkdir -p .kamal/apps/app/env/roles", output
@@ -218,8 +199,8 @@ class CliProxyTest < CliTestCase
    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("12345678")

    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2")
-      .returns(Kamal::Configuration::PROXY_MINIMUM_VERSION)
+      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :awk, "-F:", "'{print $NF}'")
+      .returns(Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION)

    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
@@ -238,7 +219,10 @@ class CliProxyTest < CliTestCase
    run_command("boot_config", "set").tap do |output|
      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
-        assert_match "Uploading \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
      end
    end
  end
@@ -248,6 +232,9 @@ class CliProxyTest < CliTestCase
      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
        assert_match "Uploading \"--log-opt max-size=10m\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
      end
    end
  end
@@ -257,6 +244,9 @@ class CliProxyTest < CliTestCase
      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
        assert_match "Uploading \"--publish 80:80 --publish 443:443 --log-opt max-size=100m\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
      end
    end
  end
@@ -266,6 +256,9 @@ class CliProxyTest < CliTestCase
      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
        assert_match "Uploading \"--publish 80:80 --publish 443:443\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
      end
    end
  end
@@ -310,19 +303,81 @@ class CliProxyTest < CliTestCase
      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
        assert_match "Uploading \"--publish 80:80 --publish 443:443 --log-opt max-size=10m --label=foo=bar --add_host=thishost:thathost\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
+      end
+    end
+  end
+
+  test "boot_config set registry" do
+    run_command("boot_config", "set", "--registry", "myreg").tap do |output|
+      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
+        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/options on #{host}", output
+        assert_match "Uploading \"myreg/basecamp/kamal-proxy\" to .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
+      end
+    end
+  end
+
+  test "boot_config set repository" do
+    run_command("boot_config", "set", "--repository", "myrepo").tap do |output|
+      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
+        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/options on #{host}", output
+        assert_match "Uploading \"myrepo/kamal-proxy\" to .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
+      end
+    end
+  end
+
+  test "boot_config set image_version" do
+    run_command("boot_config", "set", "--image_version", "0.9.9").tap do |output|
+      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
+        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Uploading \"0.9.9\" to .kamal/proxy/image_version on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/run_command on #{host}", output
+      end
+    end
+  end
+
+  test "boot_config set run_command" do
+    run_command("boot_config", "set", "--metrics_port", "9000", "--debug", "true").tap do |output|
+      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
+        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
+        assert_match "Uploading \"--publish 80:80 --publish 443:443 --log-opt max-size=10m --expose=9000\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+        assert_match "Uploading \"kamal-proxy run --debug --metrics-port \\\"9000\\\"\" to .kamal/proxy/run_command on #{host}", output
+      end
+    end
+  end
+
+  test "boot_config set all" do
+    run_command("boot_config", "set", "--docker_options", "label=foo=bar", "--registry", "myreg", "--repository", "myrepo", "--image_version", "0.9.9", "--metrics_port", "9000", "--debug", "true").tap do |output|
+      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
+        assert_match "Uploading \"--publish 80:80 --publish 443:443 --log-opt max-size=10m --expose=9000 --label=foo=bar\" to .kamal/proxy/options on #{host}", output
+        assert_match "Uploading \"myreg/myrepo/kamal-proxy\" to .kamal/proxy/image on #{host}", output
+        assert_match "Uploading \"0.9.9\" to .kamal/proxy/image_version on #{host}", output
+        assert_match "Uploading \"kamal-proxy run --debug --metrics-port \\\"9000\\\"\" to .kamal/proxy/run_command on #{host}", output
      end
    end
  end

  test "boot_config get" do
    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:cat, ".kamal/proxy/options", "||", :echo, "\"--publish 80:80 --publish 443:443 --log-opt max-size=10m\"")
-      .returns("--publish 80:80 --publish 8443:443 --label=foo=bar")
+      .with(:echo, "$(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\")")
+      .returns("--publish 80:80 --publish 8443:443 --label=foo=bar basecamp/kamal-proxy:v1.0.0")
      .twice

    run_command("boot_config", "get").tap do |output|
-      assert_match "Host 1.1.1.1: --publish 80:80 --publish 8443:443 --label=foo=bar", output
-      assert_match "Host 1.1.1.2: --publish 80:80 --publish 8443:443 --label=foo=bar", output
+      assert_match "Host 1.1.1.1: --publish 80:80 --publish 8443:443 --label=foo=bar basecamp/kamal-proxy:v1.0.0", output
+      assert_match "Host 1.1.1.2: --publish 80:80 --publish 8443:443 --label=foo=bar basecamp/kamal-proxy:v1.0.0", output
    end
  end

--- a/test/commander_test.rb
+++ b/test/commander_test.rb
@@ -149,6 +149,12 @@ class CommanderTest < ActiveSupport::TestCase
    assert_equal [], @kamal.accessory_hosts
  end

+  test "primary role hosts are first" do
+    configure_with(:deploy_with_roles_workers_primary)
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @kamal.hosts
+    assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @kamal.app_hosts
+  end
+
  private
    def configure_with(variant)
      @kamal = Kamal::Commander.new.tap do |kamal|
--- a/test/commands/app_test.rb
+++ b/test/commands/app_test.rb
@@ -497,6 +497,30 @@ class CommandsAppTest < ActiveSupport::TestCase
    ], new_command(asset_path: "/public/assets").clean_up_assets
  end

+  test "live" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy resume app-web",
+      new_command.live.join(" ")
+  end
+
+  test "maintenance" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy stop app-web",
+      new_command.maintenance.join(" ")
+  end
+
+  test "maintenance with options" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy stop app-web --drain-timeout=\"10s\" --message=\"Hi\"",
+      new_command.maintenance(drain_timeout: 10, message: "Hi").join(" ")
+  end
+
+  test "remove_proxy_app_directory" do
+    assert_equal \
+      "rm -r .kamal/proxy/apps-config/app",
+      new_command.remove_proxy_app_directory.join(" ")
+  end
+
  private
    def new_command(role: "web", host: "1.1.1.1", **additional_config)
      config = Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999")
--- a/test/commands/auditor_test.rb
+++ b/test/commands/auditor_test.rb
@@ -20,8 +20,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
    assert_equal [
      :mkdir, "-p", ".kamal", "&&",
      :echo,
-      "[#{@recorded_at}] [#{@performer}]",
-      "app removed container",
+      "\"[#{@recorded_at}] [#{@performer}] app removed container\"",
      ">>", ".kamal/app-audit.log"
    ], @auditor.record("app removed container")
  end
@@ -31,8 +30,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
      assert_equal [
        :mkdir, "-p", ".kamal", "&&",
        :echo,
-        "[#{@recorded_at}] [#{@performer}] [staging]",
-        "app removed container",
+        "\"[#{@recorded_at}] [#{@performer}] [staging] app removed container\"",
        ">>", ".kamal/app-staging-audit.log"
      ], auditor.record("app removed container")
    end
@@ -43,8 +41,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
      assert_equal [
        :mkdir, "-p", ".kamal", "&&",
        :echo,
-        "[#{@recorded_at}] [#{@performer}] [web]",
-        "app removed container",
+        "\"[#{@recorded_at}] [#{@performer}] [web] app removed container\"",
        ">>", ".kamal/app-audit.log"
      ], auditor.record("app removed container")
    end
@@ -54,8 +51,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
    assert_equal [
      :mkdir, "-p", ".kamal", "&&",
      :echo,
-      "[#{@recorded_at}] [#{@performer}] [value]",
-      "app removed container",
+      "\"[#{@recorded_at}] [#{@performer}] [value] app removed container\"",
      ">>", ".kamal/app-audit.log"
    ], @auditor.record("app removed container", detail: "value")
  end
--- a/test/commands/builder_test.rb
+++ b/test/commands/builder_test.rb
@@ -9,7 +9,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "cache" => { "type" => "gha" } })
    assert_equal "local", builder.name
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile . 2>&1",
      builder.push.join(" ")
  end

@@ -17,7 +17,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "arch" => [ "amd64" ] })
    assert_equal "local", builder.name
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile . 2>&1",
      builder.push.join(" ")
  end

@@ -25,7 +25,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "cache" => { "type" => "gha" } })
    assert_equal "local", builder.name
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile . 2>&1",
      builder.push.join(" ")
  end

@@ -33,7 +33,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "arch" => [ "amd64", "arm64" ], "remote" => "ssh://app@127.0.0.1", "cache" => { "type" => "gha" } })
    assert_equal "hybrid", builder.name
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64,linux/arm64 --builder kamal-hybrid-docker-container-ssh---app-127-0-0-1 -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/amd64,linux/arm64 --builder kamal-hybrid-docker-container-ssh---app-127-0-0-1 -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile . 2>&1",
      builder.push.join(" ")
  end

@@ -41,7 +41,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "arch" => [ "amd64", "arm64" ], "remote" => "ssh://app@127.0.0.1", "cache" => { "type" => "gha" }, "local" => false })
    assert_equal "remote", builder.name
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64,linux/arm64 --builder kamal-remote-ssh---app-127-0-0-1 -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/amd64,linux/arm64 --builder kamal-remote-ssh---app-127-0-0-1 -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile . 2>&1",
      builder.push.join(" ")
  end

@@ -49,7 +49,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "arch" => [ "#{remote_arch}" ], "remote" => "ssh://app@host", "cache" => { "type" => "gha" } })
    assert_equal "remote", builder.name
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/#{remote_arch} --builder kamal-remote-ssh---app-host -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/#{remote_arch} --builder kamal-remote-ssh---app-host -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile . 2>&1",
      builder.push.join(" ")
  end

@@ -57,7 +57,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "arch" => [ "#{local_arch}" ], "remote" => "ssh://app@host", "cache" => { "type" => "gha" } })
    assert_equal "local", builder.name
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/#{local_arch} --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/#{local_arch} --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile . 2>&1",
      builder.push.join(" ")
  end

@@ -65,7 +65,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
    builder = new_builder_command(builder: { "arch" => [ "#{local_arch}" ], "driver" => "cloud docker-org-name/builder-name" })
    assert_equal "cloud", builder.name
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/#{local_arch} --builder cloud-docker-org-name-builder-name -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/#{local_arch} --builder cloud-docker-org-name-builder-name -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile . 2>&1",
      builder.push.join(" ")
  end

@@ -112,14 +112,14 @@ class CommandsBuilderTest < ActiveSupport::TestCase
  test "build context" do
    builder = new_builder_command(builder: { "context" => ".." })
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ..",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .. 2>&1",
      builder.push.join(" ")
  end

  test "push with build args" do
    builder = new_builder_command(builder: { "args" => { "a" => 1, "b" => 2 } })
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile . 2>&1",
      builder.push.join(" ")
  end

@@ -128,7 +128,7 @@ class CommandsBuilderTest < ActiveSupport::TestCase
      FileUtils.touch("Dockerfile")
      builder = new_builder_command(builder: { "secrets" => [ "a", "b" ] })
      assert_equal \
-        "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile .",
+        "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile . 2>&1",
        builder.push.join(" ")
    end
  end
@@ -148,35 +148,35 @@ class CommandsBuilderTest < ActiveSupport::TestCase
  test "context build" do
    builder = new_builder_command(builder: { "context" => "./foo" })
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo 2>&1",
      builder.push.join(" ")
  end

  test "push with provenance" do
    builder = new_builder_command(builder: { "provenance" => "mode=max" })
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --provenance mode=max .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --provenance mode=max . 2>&1",
      builder.push.join(" ")
  end

  test "push with provenance false" do
    builder = new_builder_command(builder: { "provenance" => false })
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --provenance false .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --provenance false . 2>&1",
      builder.push.join(" ")
  end

  test "push with sbom" do
    builder = new_builder_command(builder: { "sbom" => true })
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --sbom true .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --sbom true . 2>&1",
      builder.push.join(" ")
  end

  test "push with sbom false" do
    builder = new_builder_command(builder: { "sbom" => false })
    assert_equal \
-      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --sbom false .",
+      "docker buildx build --output=type=registry --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --sbom false . 2>&1",
      builder.push.join(" ")
  end

--- a/test/commands/proxy_test.rb
+++ b/test/commands/proxy_test.rb
@@ -15,7 +15,7 @@ class CommandsProxyTest < ActiveSupport::TestCase

  test "run" do
    assert_equal \
-      "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION}",
+      "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config",
      new_command.run.join(" ")
  end

@@ -23,7 +23,7 @@ class CommandsProxyTest < ActiveSupport::TestCase
    @config.delete(:proxy)

    assert_equal \
-      "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION}",
+      "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\") $(cat .kamal/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config",
      new_command.run.join(" ")
  end

@@ -101,7 +101,7 @@ class CommandsProxyTest < ActiveSupport::TestCase

  test "version" do
    assert_equal \
-      "docker inspect kamal-proxy --format '{{.Config.Image}}' | cut -d: -f2",
+      "docker inspect kamal-proxy --format '{{.Config.Image}}' | awk -F: '{print $NF}'",
      new_command.version.join(" ")
  end

@@ -111,10 +111,28 @@ class CommandsProxyTest < ActiveSupport::TestCase
      new_command.ensure_proxy_directory.join(" ")
  end

-  test "get_boot_options" do
+  test "read_boot_options" do
    assert_equal \
-      "cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\"",
-      new_command.get_boot_options.join(" ")
+      "cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\"",
+      new_command.read_boot_options.join(" ")
+  end
+
+  test "read_image" do
+    assert_equal \
+      "cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"",
+      new_command.read_image.join(" ")
+  end
+
+  test "read_image_version" do
+    assert_equal \
+      "cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}\"",
+      new_command.read_image_version.join(" ")
+  end
+
+  test "read_run_command" do
+    assert_equal \
+      "cat .kamal/proxy/run_command 2> /dev/null || echo \"\"",
+      new_command.read_run_command.join(" ")
  end

  test "reset_boot_options" do
@@ -123,6 +141,30 @@ class CommandsProxyTest < ActiveSupport::TestCase
      new_command.reset_boot_options.join(" ")
  end

+  test "reset_image" do
+    assert_equal \
+      "rm .kamal/proxy/image",
+      new_command.reset_image.join(" ")
+  end
+
+  test "reset_image_version" do
+    assert_equal \
+      "rm .kamal/proxy/image_version",
+      new_command.reset_image_version.join(" ")
+  end
+
+  test "ensure_apps_config_directory" do
+    assert_equal \
+      "mkdir -p .kamal/proxy/apps-config",
+      new_command.ensure_apps_config_directory.join(" ")
+  end
+
+  test "reset_run_command" do
+    assert_equal \
+      "rm .kamal/proxy/run_command",
+      new_command.reset_run_command.join(" ")
+  end
+
  private
    def new_command
      Kamal::Commands::Proxy.new(Kamal::Configuration.new(@config, version: "123"))
--- a/test/configuration/accessory_test.rb
+++ b/test/configuration/accessory_test.rb
@@ -7,8 +7,8 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
      image: "dhh/app",
      registry: { "username" => "dhh", "password" => "secret" },
      servers: {
-        "web" => [ "1.1.1.1", "1.1.1.2" ],
-        "workers" => [ "1.1.1.3", "1.1.1.4" ]
+        "web" => [ { "1.1.1.1" => "writer" }, { "1.1.1.2" => "reader" } ],
+        "workers" => [ { "1.1.1.3" => "writer" }, "1.1.1.4" ]
      },
      builder: { "arch" => "amd64" },
      env: { "REDIS_URL" => "redis://x/y" },
@@ -55,7 +55,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
          "service" => "custom-monitoring",
          "image" => "monitoring:latest",
          "registry" => { "server" => "other.registry", "username" => "user", "password" => "pw" },
-          "roles" => [ "web" ],
+          "role" => "web",
          "port" => "4321:4321",
          "labels" => {
            "cache" => "true"
@@ -70,6 +70,14 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
          "proxy" => {
            "host" => "monitoring.example.com"
          }
+        },
+        "proxy" => {
+          "image" => "proxy:latest",
+          "tags" => [ "writer", "reader" ]
+        },
+        "logger" => {
+          "image" => "logger:latest",
+          "tag" => "writer"
        }
      }
    }
@@ -107,6 +115,8 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
    assert_equal [ "1.1.1.5" ], @config.accessory(:mysql).hosts
    assert_equal [ "1.1.1.6", "1.1.1.7" ], @config.accessory(:redis).hosts
    assert_equal [ "1.1.1.1", "1.1.1.2" ], @config.accessory(:monitoring).hosts
+    assert_equal [ "1.1.1.1", "1.1.1.3", "1.1.1.2" ], @config.accessory(:proxy).hosts
+    assert_equal [ "1.1.1.1", "1.1.1.3" ], @config.accessory(:logger).hosts
  end

  test "missing host" do
@@ -117,14 +127,14 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
    end
  end

-  test "setting host, hosts and roles" do
+  test "setting host, hosts, roles and tags" do
    @deploy[:accessories]["mysql"]["hosts"] = [ "mysql-db1" ]
    @deploy[:accessories]["mysql"]["roles"] = [ "db" ]

    exception = assert_raises(Kamal::ConfigurationError) do
      Kamal::Configuration.new(@deploy)
    end
-    assert_equal "accessories/mysql: specify one of `host`, `hosts` or `roles`", exception.message
+    assert_equal "accessories/mysql: specify one of `host`, `hosts`, `role`, `roles`, `tag` or `tags`", exception.message
  end

  test "all hosts" do
@@ -187,4 +197,12 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
    assert @config.accessory(:monitoring).running_proxy?
    assert_equal [ "monitoring.example.com" ], @config.accessory(:monitoring).proxy.hosts
  end
+
+  test "can't set restart in options" do
+    @deploy[:accessories]["mysql"]["options"] = { "restart" => "always" }
+
+    assert_raises Kamal::ConfigurationError, "servers/workers: Cannot set restart policy in docker options, unless-stopped is required" do
+      Kamal::Configuration.new(@deploy)
+    end
+  end
 end
--- a/test/configuration/env/tags_test.rb
+++ b/test/configuration/env/tags_test.rb
@@ -92,7 +92,25 @@ class ConfigurationEnvTagsTest < ActiveSupport::TestCase
      }

      config = Kamal::Configuration.new(deploy)
-      assert_equal "hello", config.role("web").env("1.1.1.1").secrets["PASSWORD"]
+      assert_equal "PASSWORD=hello\n", config.role("web").env("1.1.1.1").secrets_io.string
+    end
+  end
+
+  test "aliased tag secret env" do
+    with_test_secrets("secrets" => "PASSWORD=hello\nALIASED_PASSWORD=aliased_hello") do
+      deploy = {
+        service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+        servers: [ { "1.1.1.1" => "secrets" } ],
+        builder: { "arch" => "amd64" },
+        env: {
+          "tags" => {
+            "secrets" => { "secret" => [ "PASSWORD:ALIASED_PASSWORD" ] }
+          }
+        }
+      }
+
+      config = Kamal::Configuration.new(deploy)
+      assert_equal "PASSWORD=aliased_hello\n", config.role("web").env("1.1.1.1").secrets_io.string
    end
  end

--- a/test/configuration/env_test.rb
+++ b/test/configuration/env_test.rb
@@ -48,6 +48,20 @@ class ConfigurationEnvTest < ActiveSupport::TestCase
    end
  end

+  test "aliased secrets" do
+    with_test_secrets("secrets" => "ALIASED_PASSWORD=hello") do
+      config = {
+        "secret" => [ "PASSWORD:ALIASED_PASSWORD" ],
+        "clear" => {}
+      }
+
+      assert_config \
+        config: config,
+        clear: {},
+        secrets: { "PASSWORD" => "hello" }
+    end
+  end
+
  private
    def assert_config(config:, clear: {}, secrets: {})
      env = Kamal::Configuration::Env.new config: config, secrets: Kamal::Secrets.new
--- a/test/configuration/proxy/boot_test.rb
+++ b/test/configuration/proxy/boot_test.rb
@@ -0,0 +1,29 @@
+require "test_helper"
+
+class ConfigurationProxyBootTest < ActiveSupport::TestCase
+  setup do
+    ENV["RAILS_MASTER_KEY"] = "456"
+    ENV["VERSION"] = "missing"
+
+    @deploy = {
+      service: "app", image: "dhh/app",
+      registry: { "username" => "dhh", "password" => "secret" },
+      builder: { "arch" => "amd64" },
+      env: { "REDIS_URL" => "redis://x/y" },
+      servers: [ "1.1.1.1", "1.1.1.2" ],
+      volumes: [ "/local/path:/container/path" ]
+    }
+
+    @config = Kamal::Configuration.new(@deploy)
+    @proxy_boot_config = @config.proxy_boot
+  end
+
+  test "proxy directories" do
+    assert_equal ".kamal/proxy/apps-config", @proxy_boot_config.apps_directory
+    assert_equal "/home/kamal-proxy/.apps-config", @proxy_boot_config.apps_container_directory
+    assert_equal ".kamal/proxy/apps-config/app", @proxy_boot_config.app_directory
+    assert_equal "/home/kamal-proxy/.apps-config/app", @proxy_boot_config.app_container_directory
+    assert_equal ".kamal/proxy/apps-config/app/error_pages", @proxy_boot_config.error_pages_directory
+    assert_equal "/home/kamal-proxy/.apps-config/app/error_pages", @proxy_boot_config.error_pages_container_directory
+  end
+end
--- a/test/configuration/proxy_test.rb
+++ b/test/configuration/proxy_test.rb
@@ -38,6 +38,13 @@ class ConfigurationProxyTest < ActiveSupport::TestCase
    assert_not config.proxy.ssl?
  end

+  test "false not allowed" do
+    @deploy[:proxy] = false
+    assert_raises(Kamal::ConfigurationError, "proxy: should be a hash") do
+      config.proxy
+    end
+  end
+
  private
    def config
      Kamal::Configuration.new(@deploy)
--- a/test/configuration/role_test.rb
+++ b/test/configuration/role_test.rb
@@ -258,6 +258,14 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
    assert_equal "18s", config_with_roles.role(:workers).proxy.deploy_options[:"target-timeout"]
  end

+  test "can't set restart in options" do
+    @deploy_with_roles[:servers]["workers"]["options"] = { "restart" => "always" }
+
+    assert_raises Kamal::ConfigurationError, "servers/workers: Cannot set restart policy in docker options, unless-stopped is required" do
+      Kamal::Configuration.new(@deploy_with_roles)
+    end
+  end
+
  private
    def config
      Kamal::Configuration.new(@deploy)
--- a/test/configuration_test.rb
+++ b/test/configuration_test.rb
@@ -30,7 +30,7 @@ class ConfigurationTest < ActiveSupport::TestCase
  %i[ service image registry ].each do |key|
    test "#{key} config required" do
      assert_raise(Kamal::ConfigurationError) do
-        Kamal::Configuration.new @deploy.tap { _1.delete key }
+        Kamal::Configuration.new @deploy.tap { |config| config.delete key }
      end
    end
  end
@@ -38,21 +38,36 @@ class ConfigurationTest < ActiveSupport::TestCase
  %w[ username password ].each do |key|
    test "registry #{key} required" do
      assert_raise(Kamal::ConfigurationError) do
-        Kamal::Configuration.new @deploy.tap { _1[:registry].delete key }
+        Kamal::Configuration.new @deploy.tap { |config| config[:registry].delete key }
      end
    end
  end

  test "service name valid" do
    assert_nothing_raised do
-      Kamal::Configuration.new(@deploy.tap { _1[:service] = "hey-app1_primary" })
-      Kamal::Configuration.new(@deploy.tap { _1[:service] = "MyApp" })
+      Kamal::Configuration.new(@deploy.tap { |config| config[:service] = "hey-app1_primary" })
+      Kamal::Configuration.new(@deploy.tap { |config| config[:service] = "MyApp" })
    end
  end

  test "service name invalid" do
    assert_raise(Kamal::ConfigurationError) do
-      Kamal::Configuration.new @deploy.tap { _1[:service] = "app.com" }
+      Kamal::Configuration.new @deploy.tap { |config| config[:service] = "app.com" }
+    end
+  end
+
+  test "servers required" do
+    assert_raise(Kamal::ConfigurationError) do
+      Kamal::Configuration.new @deploy.tap { |config| config.delete(:servers) }
+    end
+  end
+
+  test "servers not required with accessories" do
+    assert_nothing_raised do
+      @deploy.delete(:servers)
+      @deploy[:accessories] = { "foo" => { "image" => "foo/bar", "host" => "1.1.1.1" } }
+
+      Kamal::Configuration.new(@deploy)
    end
  end

@@ -250,7 +265,7 @@ class ConfigurationTest < ActiveSupport::TestCase
  test "destination required" do
    dest_config_file = Pathname.new(File.expand_path("fixtures/deploy_for_required_dest.yml", __dir__))

-    assert_raises(Kamal::ConfigurationError) do
+    assert_raises(ArgumentError, "You must specify a destination") do
      config = Kamal::Configuration.create_from config_file: dest_config_file
    end

--- a/test/fixtures/deploy_with_error_pages.yml
+++ b/test/fixtures/deploy_with_error_pages.yml
@@ -0,0 +1,11 @@
+service: app
+image: dhh/app
+servers:
+  - "1.1.1.1"
+  - "1.1.1.2"
+registry:
+  username: user
+  password: pw
+builder:
+  arch: amd64
+error_pages_path: public
--- a/test/fixtures/deploy_with_only_workers.yml
+++ b/test/fixtures/deploy_with_only_workers.yml
--- a/test/fixtures/deploy_with_roles_workers_primary.yml
+++ b/test/fixtures/deploy_with_roles_workers_primary.yml
@@ -0,0 +1,19 @@
+service: app
+image: dhh/app
+servers:
+  workers:
+    - 1.1.1.1
+    - 1.1.1.2
+  web:
+    - 1.1.1.3
+    - 1.1.1.4
+env:
+  REDIS_URL: redis://x/y
+registry:
+  server: registry.digitalocean.com
+  username: user
+  password: pw
+builder:
+  arch: amd64
+deploy_timeout: 1
+primary_role: workers
--- a/test/integration/accessory_test.rb
+++ b/test/integration/accessory_test.rb
@@ -17,10 +17,39 @@ class AccessoryTest < IntegrationTest
    logs = kamal :accessory, :logs, :busybox, capture: true
    assert_match /Starting busybox.../, logs

+    boot = kamal :accessory, :boot, :busybox, capture: true
+    assert_match /Skipping booting `busybox` on vm1, vm2, a container already exists/, boot
+
    kamal :accessory, :remove, :busybox, "-y"
    assert_accessory_not_running :busybox
  end

+  test "proxied: boot, stop, start, restart, logs, remove" do
+    @app = "app_with_proxied_accessory"
+
+    kamal :proxy, :boot
+
+    kamal :accessory, :boot, :netcat
+    assert_accessory_running :netcat
+    assert_netcat_is_up
+
+    kamal :accessory, :stop, :netcat
+    assert_accessory_not_running :netcat
+    assert_netcat_not_found
+
+    kamal :accessory, :start, :netcat
+    assert_accessory_running :netcat
+    assert_netcat_is_up
+
+    kamal :accessory, :restart, :netcat
+    assert_accessory_running :netcat
+    assert_netcat_is_up
+
+    kamal :accessory, :remove, :netcat, "-y"
+    assert_accessory_not_running :netcat
+    assert_netcat_not_found
+  end
+
  private
    def assert_accessory_running(name)
      assert_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
@@ -33,4 +62,25 @@ class AccessoryTest < IntegrationTest
    def accessory_details(name)
      kamal :accessory, :details, name, capture: true
    end
+
+    def assert_netcat_is_up
+      response = netcat_response
+      debug_response_code(response, "200")
+      assert_equal "200", response.code
+    end
+
+    def assert_netcat_not_found
+      response = netcat_response
+      debug_response_code(response, "404")
+      assert_equal "404", response.code
+    end
+
+    def netcat_response
+      uri = URI.parse("http://127.0.0.1:12345/up")
+      http = Net::HTTP.new(uri.host, uri.port)
+      request = Net::HTTP::Get.new(uri)
+      request["Host"] = "netcat"
+
+      http.request(request)
+    end
 end
--- a/test/integration/app_test.rb
+++ b/test/integration/app_test.rb
@@ -48,9 +48,36 @@ class AppTest < IntegrationTest
    assert_match "App Host: vm1", exec_output
    assert_match /1 root      0:\d\d nginx/, exec_output

+    kamal :app, :maintenance
+    assert_app_in_maintenance
+
+    kamal :app, :live
+    assert_app_is_up
+
    kamal :app, :remove

    assert_app_not_found
    assert_app_directory_removed
  end
+
+  test "custom error pages" do
+    @app = "app_with_roles"
+
+    kamal :deploy
+    assert_app_is_up
+
+    kamal :app, :maintenance
+    assert_app_in_maintenance message: "Custom Maintenance Page"
+
+    kamal :app, :live
+    kamal :app, :maintenance, "--message", "\"Testing Maintence Mode\""
+    assert_app_in_maintenance message: "Custom Maintenance Page: Testing Maintence Mode"
+
+    second_version = update_app_rev
+
+    kamal :redeploy
+
+    kamal :app, :maintenance
+    assert_app_in_maintenance message: "Custom Maintenance Page"
+  end
 end
--- a/test/integration/docker/deployer/app/.kamal/hooks/pre-connect
+++ b/test/integration/docker/deployer/app/.kamal/hooks/pre-connect
@@ -1,4 +1,5 @@
 #!/bin/sh

 echo "About to lock..."
+env
 mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-connect
--- a/test/integration/docker/deployer/app/.kamal/hooks/pre-deploy
+++ b/test/integration/docker/deployer/app/.kamal/hooks/pre-deploy
@@ -1,3 +1,6 @@
 #!/bin/sh
+set -e
+
+kamal proxy boot_config set --registry registry:4443
 echo "Deployed!"
 mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-deploy
--- a/test/integration/docker/deployer/app/config/deploy.yml
+++ b/test/integration/docker/deployer/app/config/deploy.yml
@@ -41,3 +41,8 @@ accessories:
    cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done'
    roles:
      - web
+  busybox2:
+    service: custom-busybox
+    image: registry:4443/busybox:1.36.0
+    cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done'
+    host: vm3
--- a/test/integration/docker/deployer/app_with_proxied_accessory/.kamal/hooks/pre-deploy
+++ b/test/integration/docker/deployer/app_with_proxied_accessory/.kamal/hooks/pre-deploy
@@ -0,0 +1,4 @@
+#!/bin/sh
+set -e
+
+kamal proxy boot_config set --registry registry:4443
--- a/test/integration/docker/deployer/app_with_proxied_accessory/config/deploy.yml
+++ b/test/integration/docker/deployer/app_with_proxied_accessory/config/deploy.yml
@@ -1,7 +1,5 @@
 service: app_with_proxied_accessory
 image: app_with_proxied_accessory
-servers:
-  - vm1
 env:
  clear:
    CLEAR_TOKEN: 4321
@@ -24,15 +22,13 @@ accessories:
    service: custom-busybox
    image: registry:4443/busybox:1.36.0
    cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done'
-    roles:
-      - web
+    host: vm1
  netcat:
    service: netcat
    image: registry:4443/busybox:1.36.0
    cmd: >
      sh -c 'echo "Starting netcat..."; while true; do echo -e "HTTP/1.1 200 OK\r\nContent-Length: 11\r\n\r\nHello Ruby" | nc -l -p 80; done'
-    roles:
-      - web
+    host: vm1
    port: 12345:80
    proxy:
      host: netcat
@@ -41,4 +37,4 @@ accessories:
        interval: 1
        timeout: 1
        path: "/"
-
+drain_timeout: 2
--- a/test/integration/docker/deployer/app_with_roles/.kamal/hooks/pre-deploy
+++ b/test/integration/docker/deployer/app_with_roles/.kamal/hooks/pre-deploy
@@ -1,3 +1,6 @@
 #!/bin/sh
+set -e
+
+kamal proxy boot_config set --registry registry:4443
 echo "Deployed!"
 mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-deploy
--- a/test/integration/docker/deployer/app_with_roles/config/deploy.yml
+++ b/test/integration/docker/deployer/app_with_roles/config/deploy.yml
@@ -37,6 +37,7 @@ proxy:
      - X-Request-Start

 asset_path: /usr/share/nginx/html/versions
+error_pages_path: error_pages

 registry:
  server: registry:4443
--- a/test/integration/docker/deployer/app_with_roles/error_pages/503.html
+++ b/test/integration/docker/deployer/app_with_roles/error_pages/503.html
@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>503 Service Interrupted</title>
+  </head>
+  <body>
+    <p>Custom Maintenance Page: {{ .Message }}</p>
+  </body>
+</html>
--- a/test/integration/docker/deployer/app_with_traefik/.kamal/hooks/pre-deploy
+++ b/test/integration/docker/deployer/app_with_traefik/.kamal/hooks/pre-deploy
@@ -1,3 +1,7 @@
-kamal proxy boot_config set --publish false \
+set -e
+
+kamal proxy boot_config set --registry registry:4443 \
+                            --publish false \
                            --docker_options label=traefik.http.services.kamal_proxy.loadbalancer.server.scheme=http \
-                                             label=traefik.http.routers.kamal_proxy.rule=PathPrefix\(\`/\`\)
+                                             label=traefik.http.routers.kamal_proxy.rule=PathPrefix\(\`/\`\) \
+                                             sysctl=net.ipv4.ip_local_port_range=\"10000\ 60999\"
--- a/test/integration/docker/deployer/setup.sh
+++ b/test/integration/docker/deployer/setup.sh
@@ -20,6 +20,7 @@ push_image_to_registry_4443() {
 install_kamal
 push_image_to_registry_4443 nginx 1-alpine-slim
 push_image_to_registry_4443 busybox 1.36.0
+push_image_to_registry_4443 basecamp/kamal-proxy v0.9.0

 # .ssh is on a shared volume that persists between runs. Clean it up as the
 # churn of temporary vm IPs can eventually create conflicts.
--- a/test/integration/docker/registry/Dockerfile
+++ b/test/integration/docker/registry/Dockerfile
@@ -1,4 +1,4 @@
-FROM registry
+FROM registry:3

 COPY boot.sh .

--- a/test/integration/docker/registry/boot.sh
+++ b/test/integration/docker/registry/boot.sh
@@ -2,4 +2,4 @@

 while [ ! -f /certs/domain.crt ]; do sleep 1; done

-exec /entrypoint.sh /etc/docker/registry/config.yml
+exec /entrypoint.sh /etc/distribution/config.yml
--- a/test/integration/integration_test.rb
+++ b/test/integration/integration_test.rb
@@ -11,7 +11,7 @@ class IntegrationTest < ActiveSupport::TestCase
  end

  teardown do
-    unless passed?
+    if !passed? && ENV["DEBUG_CONTAINER_LOGS"]
      [ :deployer, :vm1, :vm2, :shared, :load_balancer, :registry ].each do |container|
        puts
        puts "Logs for #{container}:"
@@ -25,8 +25,8 @@ class IntegrationTest < ActiveSupport::TestCase
    def docker_compose(*commands, capture: false, raise_on_error: true)
      command = "TEST_ID=#{ENV["TEST_ID"]} docker compose #{commands.join(" ")}"
      succeeded = false
-      if capture
-        result = stdouted { succeeded = system("cd test/integration && #{command}") }
+      if capture || !ENV["DEBUG"]
+        result = stdouted { stderred { succeeded = system("cd test/integration && #{command}") } }
      else
        succeeded = system("cd test/integration && #{command}")
      end
@@ -45,15 +45,22 @@ class IntegrationTest < ActiveSupport::TestCase
    end

    def assert_app_is_down
-      response = app_response
-      debug_response_code(response, "502")
-      assert_equal "502", response.code
+      assert_app_error_code("502")
+    end
+
+    def assert_app_in_maintenance(message: nil)
+      assert_app_error_code("503", message: message)
    end

    def assert_app_not_found
+      assert_app_error_code("404")
+    end
+
+    def assert_app_error_code(code, message: nil)
      response = app_response
-      debug_response_code(response, "404")
-      assert_equal "404", response.code
+      debug_response_code(response, code)
+      assert_equal code, response.code
+      assert_match message, response.body.strip if message
    end

    def assert_app_is_up(version: nil, app: @app)
--- a/test/integration/main_test.rb
+++ b/test/integration/main_test.rb
@@ -9,8 +9,12 @@ class MainTest < IntegrationTest
    kamal :deploy
    assert_app_is_up version: first_version
    assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "pre-app-boot", "post-app-boot", "post-deploy"
+
    assert_envs version: first_version

+    output = kamal :app, :exec, "--verbose", "ls", "-r", "web", capture: true
+    assert_hook_env_variables output, version: first_version
+
    second_version = update_app_rev

    kamal :redeploy
@@ -28,7 +32,7 @@ class MainTest < IntegrationTest
    assert_match /Proxy Host: vm2/, details
    assert_match /App Host: vm1/, details
    assert_match /App Host: vm2/, details
-    assert_match /basecamp\/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION}/, details
+    assert_match /basecamp\/kamal-proxy:#{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}/, details
    assert_match /registry:4443\/app:#{first_version}/, details

    audit = kamal :audit, capture: true
@@ -60,7 +64,7 @@ class MainTest < IntegrationTest
    version = latest_app_version

    assert_equal [ "web" ], config[:roles]
-    assert_equal [ "vm1", "vm2" ], config[:hosts]
+    assert_equal [ "vm1", "vm2", "vm3" ], config[:hosts]
    assert_equal "vm1", config[:primary_host]
    assert_equal version, config[:version]
    assert_equal "registry:4443/app", config[:repository]
@@ -88,8 +92,6 @@ class MainTest < IntegrationTest
  end

  test "setup and remove" do
-    @app = "app_with_roles"
-
    kamal :proxy, :boot_config, "set",
      "--publish=false",
      "--docker-options=label=traefik.http.services.kamal_proxy.loadbalancer.server.scheme=http",
@@ -172,21 +174,36 @@ class MainTest < IntegrationTest
      assert_equal "200", Net::HTTP.get_response(URI.parse("http://#{app_host}:12345/versions/.hidden")).code
    end

-    def vm1_image_ids
-      docker_compose("exec vm1 docker image ls -q", capture: true).strip.split("\n")
+    def image_ids(vm:)
+      docker_compose("exec #{vm} docker image ls -q", capture: true).strip.split("\n")
    end

-    def vm1_container_ids
-      docker_compose("exec vm1 docker ps -a -q", capture: true).strip.split("\n")
+    def container_ids(vm:)
+      docker_compose("exec #{vm} docker ps -a -q", capture: true).strip.split("\n")
    end

    def assert_no_images_or_containers
-      assert vm1_image_ids.empty?
-      assert vm1_container_ids.empty?
+      [ :vm1, :vm2, :vm3 ].each do |vm|
+        assert image_ids(vm: vm).empty?
+        assert container_ids(vm: vm).empty?
+      end
    end

    def assert_images_and_containers
-      assert vm1_image_ids.any?
-      assert vm1_container_ids.any?
+      [ :vm1, :vm2, :vm3 ].each do |vm|
+        assert image_ids(vm: vm).any?
+        assert container_ids(vm: vm).any?
+      end
+    end
+
+    def assert_hook_env_variables(output, version:)
+      assert_match "KAMAL_VERSION=#{version}", output
+      assert_match "KAMAL_SERVICE=app", output
+      assert_match "KAMAL_SERVICE_VERSION=app@#{version[0..6]}", output
+      assert_match "KAMAL_COMMAND=app", output
+      assert_match "KAMAL_PERFORMER=deployer@example.com", output
+      assert_match /KAMAL_RECORDED_AT=\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ/, output
+      assert_match "KAMAL_HOSTS=vm1,vm2", output
+      assert_match "KAMAL_ROLES=web", output
    end
 end
--- a/test/integration/proxied_accessory_test.rb
+++ b/test/integration/proxied_accessory_test.rb
@@ -1,63 +0,0 @@
-require_relative "integration_test"
-
-class ProxiedAccessoryTest < IntegrationTest
-  test "boot, stop, start, restart, logs, remove" do
-    @app = "app_with_proxied_accessory"
-
-    kamal :deploy
-
-    kamal :accessory, :boot, :netcat
-    assert_accessory_running :netcat
-    assert_netcat_is_up
-
-    kamal :accessory, :stop, :netcat
-    assert_accessory_not_running :netcat
-    assert_netcat_not_found
-
-    kamal :accessory, :start, :netcat
-    assert_accessory_running :netcat
-    assert_netcat_is_up
-
-    kamal :accessory, :restart, :netcat
-    assert_accessory_running :netcat
-    assert_netcat_is_up
-
-    kamal :accessory, :remove, :netcat, "-y"
-    assert_accessory_not_running :netcat
-    assert_netcat_not_found
-  end
-
-  private
-    def assert_accessory_running(name)
-      assert_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
-    end
-
-    def assert_accessory_not_running(name)
-      assert_no_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
-    end
-
-    def accessory_details(name)
-      kamal :accessory, :details, name, capture: true
-    end
-
-    def assert_netcat_is_up
-      response = netcat_response
-      debug_response_code(response, "200")
-      assert_equal "200", response.code
-    end
-
-    def assert_netcat_not_found
-      response = netcat_response
-      debug_response_code(response, "404")
-      assert_equal "404", response.code
-    end
-
-    def netcat_response
-      uri = URI.parse("http://127.0.0.1:12345/up")
-      http = Net::HTTP.new(uri.host, uri.port)
-      request = Net::HTTP::Get.new(uri)
-      request["Host"] = "netcat"
-
-      http.request(request)
-    end
-end
--- a/test/integration/proxy_test.rb
+++ b/test/integration/proxy_test.rb
@@ -6,6 +6,8 @@ class ProxyTest < IntegrationTest
  end

  test "boot, reboot, stop, start, restart, logs, remove" do
+    kamal :proxy, :boot_config, :set, "--registry", "registry:4443"
+
    kamal :proxy, :boot
    assert_proxy_running

@@ -46,7 +48,27 @@ class ProxyTest < IntegrationTest
    logs = kamal :proxy, :logs, capture: true
    assert_match /No previous state to restore/, logs

+    kamal :proxy, :boot_config, :set, "--registry", "registry:4443", "--docker-options='sysctl net.ipv4.ip_local_port_range=\"10000 60999\"'"
+    assert_docker_options_in_file
+
+    kamal :proxy, :reboot, "-y"
+    assert_docker_options_in_container
+
+    kamal :proxy, :boot_config, :reset
+
    kamal :proxy, :remove
    assert_proxy_not_running
  end
+
+  private
+    def assert_docker_options_in_file
+      boot_config = kamal :proxy, :boot_config, :get, capture: true
+      assert_match "Host vm1: --publish 80:80 --publish 443:443 --log-opt max-size=10m --sysctl net.ipv4.ip_local_port_range=\"10000 60999\"", boot_config
+    end
+
+    def assert_docker_options_in_container
+      assert_equal \
+        "{\"net.ipv4.ip_local_port_range\":\"10000 60999\"}",
+        docker_compose("exec vm1 docker inspect --format '{{ json .HostConfig.Sysctls }}' kamal-proxy", capture: true).strip
+    end
 end
--- a/test/secrets/aws_secrets_manager_adapter_test.rb
+++ b/test/secrets/aws_secrets_manager_adapter_test.rb
@@ -4,7 +4,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
  test "fails when errors are present" do
    stub_ticks.with("aws --version 2> /dev/null")
    stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list unknown1 unknown2 --profile default")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list unknown1 unknown2 --profile default --output json")
      .returns(<<~JSON)
        {
          "SecretValues": [],
@@ -33,7 +33,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
  test "fetch" do
    stub_ticks.with("aws --version 2> /dev/null")
    stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 secret2/KEY3 --profile default")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 secret2/KEY3 --profile default --output json")
      .returns(<<~JSON)
        {
          "SecretValues": [
@@ -76,7 +76,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
  test "fetch with string value" do
    stub_ticks.with("aws --version 2> /dev/null")
    stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret secret2/KEY1 --profile default")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret secret2/KEY1 --profile default --output json")
      .returns(<<~JSON)
        {
          "SecretValues": [
@@ -118,7 +118,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
  test "fetch with secret names" do
    stub_ticks.with("aws --version 2> /dev/null")
    stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 --profile default")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 --profile default --output json")
      .returns(<<~JSON)
        {
          "SecretValues": [
@@ -159,7 +159,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
  test "fetch without account option omits --profile" do
    stub_ticks.with("aws --version 2> /dev/null")
    stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 --output json")
      .returns(<<~JSON)
        {
          "SecretValues": [
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -58,9 +58,7 @@ class ActiveSupport::TestCase
    def setup_test_secrets(**files)
      @original_pwd = Dir.pwd
      @secrets_tmpdir = Dir.mktmpdir
-      fixtures_dup = File.join(@secrets_tmpdir, "test")
-      FileUtils.mkdir_p(fixtures_dup)
-      FileUtils.cp_r("test/fixtures/", fixtures_dup)
+      copy_fixtures(@secrets_tmpdir)

      Dir.chdir(@secrets_tmpdir)
      FileUtils.mkdir_p(".kamal")
@@ -75,6 +73,30 @@ class ActiveSupport::TestCase
      Dir.chdir(@original_pwd)
      FileUtils.rm_rf(@secrets_tmpdir)
    end
+
+    def with_error_pages(directory:)
+      error_pages_tmpdir = Dir.mktmpdir
+
+      Dir.mktmpdir do |tmpdir|
+        copy_fixtures(tmpdir)
+
+        Dir.chdir(tmpdir) do
+          FileUtils.mkdir_p(directory)
+          Dir.chdir(directory) do
+            File.write("404.html", "404 page")
+            File.write("503.html", "503 page")
+          end
+
+          yield
+        end
+      end
+    end
+
+    def copy_fixtures(to_dir)
+      new_test_dir = File.join(to_dir, "test")
+      FileUtils.mkdir_p(new_test_dir)
+      FileUtils.cp_r("test/fixtures/", new_test_dir)
+    end
 end

 class SecretAdapterTestCase < ActiveSupport::TestCase
Author	SHA1	Message	Date
Donal McBreen	22e7243b10	Bump version for 2.6.1	2025-05-15 15:15:29 +01:00
Donal McBreen	259a018d5a	Merge pull request #1558 from basecamp/per-role-proxy-docs Per role proxy docs	2025-05-15 15:00:11 +01:00
Donal McBreen	a82e88d5c9	Merge pull request #1560 from basecamp/dont-redeploy-on-proxy-reboot Don't deploy on proxy reboot	2025-05-15 14:57:58 +01:00
Donal McBreen	d6459e869a	Merge pull request #1559 from basecamp/default-proxy-config-if-nil Default the proxy config if it is nil	2025-05-15 14:47:46 +01:00
Donal McBreen	ad21c7e984	Don't deploy on proxy reboot It shouldn't be necessary to deploy the app on proxy reboot. When there are multiple apps using the same proxy we'll only deploy the one we run the reboot command from, so we don't always reboot anyway.	2025-05-15 14:45:19 +01:00
Donal McBreen	87965281a3	Default the proxy config is it is nil Instead of checking for the proxy key, we'll set the config to {} if it is nil in the Kamal::Configuration::Proxy initializer. This is a bit cleaner, and maybe it will help with https://github.com/basecamp/kamal/issues/1555 if somehow @raw_config.key?(:proxy) is false but @raw_config.proxy is not nil.	2025-05-15 14:33:05 +01:00
Donal McBreen	dca96eafaa	Merge pull request #1557 from basecamp/sort-primary-role-app-hosts-first Ensure primary_role app hosts are sorted first	2025-05-15 10:16:21 +01:00
Donal McBreen	7b1439c3c6	Update per-role proxy docs Clarify that proxy: true/proxy: false only belong in the role config, not at the root level.	2025-05-15 10:14:52 +01:00
Donal McBreen	b9e5ce7ca7	Ensure primary_role app hosts are sorted first When booting non-primary role hosts we will always wait for a primary role host to boor first. So when booting in groups, if there are no primary role hosts in the first batch, then booting will stall. Sort primary role app_hosts first to avoid this. Fixes: https://github.com/basecamp/kamal/issues/1553	2025-05-15 09:51:40 +01:00
Donal McBreen	f62c1a50c4	Merge pull request #1554 from basecamp/pre-connect-hook-before-remote-builds Run pre-connect hooks before building	2025-05-14 16:05:53 +01:00
Donal McBreen	2c1d6ed891	Run pre-connect hooks before building They might be needed for remote builds or the pre-build hook.	2025-05-14 15:55:54 +01:00
Donal McBreen	031f55ecf7	Bump version for 2.6.0	2025-05-13 09:50:52 +01:00
Donal McBreen	d98d6a3475	Merge pull request #1550 from krzysztoff1/add-singular-role Add a singular role	2025-05-12 15:20:55 +01:00
Krzysztof Duda	78c9d610cf	Add a singular role	2025-05-12 11:07:10 +02:00
Donal McBreen	4187ee2397	Merge pull request #1547 from basecamp/pin-accessories-to-tags Pin accessories to tags	2025-05-12 08:48:52 +01:00
David Heinemeier Hansson	7bfb2ed9f2	Actually test the fixture for singular	2025-05-09 21:50:07 +02:00
David Heinemeier Hansson	299c741c1b	More natural api when you are just applying accessory to a single tag	2025-05-09 21:47:26 +02:00
David Heinemeier Hansson	fb82d04aaf	Use #filter_map instead of #collect + #compact	2025-05-09 21:30:33 +02:00
David Heinemeier Hansson	9d5a534ef8	Refactored for clarity and style	2025-05-09 21:26:41 +02:00
David Heinemeier Hansson	5ad000a08e	Unnecessary parenthesis	2025-05-09 21:16:31 +02:00
David Heinemeier Hansson	1ca2b4d394	Test with multiple host matches across roles	2025-05-09 21:15:44 +02:00
David Heinemeier Hansson	9aac51bbd0	Extract hosts for accessories by tags	2025-05-09 21:11:28 +02:00
David Heinemeier Hansson	83a5636e27	Pin accessories to tags	2025-05-09 18:14:47 +02:00
Donal McBreen	2d43f788c4	Merge pull request #1546 from basecamp/min-proxy-version-0.9.0 Set minimum proxy version to 0.9.0	2025-05-09 09:13:23 +01:00
Donal McBreen	c351c2d2de	Set minimum proxy version to 0.9.0	2025-05-09 08:40:41 +01:00
Donal McBreen	0d36fc4bd0	Merge pull request #1536 from lukef/bump-ed25519 Bumping ed25519 dependency to fix compile errors	2025-05-09 08:17:44 +01:00
Donal McBreen	317f00281a	Merge pull request #1504 from basecamp/proxy-metrics-port Allow kamal-proxy run command options to be set	2025-05-06 13:15:47 +01:00
Donal McBreen	226e7091db	Add run_command_file to proxy boot	2025-05-06 12:14:58 +01:00
Donal McBreen	e32ea2e276	Expose the metrics port	2025-05-06 12:12:37 +01:00
Donal McBreen	1ea5d0bd86	Allow kamal-proxy run command options to be set Allow --metrics_port and --debug options to be set via the boot config. --metrics_port support will come in kamal-proxy v0.8.8, so this option doesn't work right now. This will be updated before the next Kamal release though and we can add integration tests for the metrics at that point.	2025-05-06 12:11:48 +01:00
Luke Freeman	f5f1bab8bf	bumping ed25519 dependency to fix compile errors	2025-05-05 08:29:50 -07:00
Donal McBreen	419a1171fa	Merge pull request #1528 from rahearn/update-hooks-documentation Update name of KAMAL_ROLES in sample hooks files	2025-04-28 08:12:25 +01:00
Ryan Ahearn	2f7feaf59d	Update name of KAMAL_ROLES in sample hooks files	2025-04-26 12:17:59 -04:00
Donal McBreen	52c6191803	Merge pull request #1526 from basecamp/proxy-boot-config Extract Kamal::Configuration::Proxy::Boot	2025-04-24 08:44:55 +01:00
Donal McBreen	b1c5c5092f	Fix polynomial regexp issue	2025-04-24 08:17:02 +01:00
Donal McBreen	128294672d	Extract Kamal::Configuration::Proxy::Boot This is for boot time configuration for the kamal proxy. Config in here doesn't not belong in Kamal::Configuration::Proxy which is for deploy time configuration for the app itself. Kamal apps don't contain boot time config, because multiple apps can share a proxy and the config could conflict.	2025-04-23 16:16:12 +01:00
Donal McBreen	eb915f830e	Merge pull request #1522 from basecamp/create-directories-before-mapping Create the .kamal/proxy/apps-config directory	2025-04-22 15:29:36 +01:00
Donal McBreen	d26b3f1768	Create the .kamal/proxy/apps-config directory Manually create it to avoid ownership issues when docker creates it for you.	2025-04-22 15:18:54 +01:00
Donal McBreen	8789a1b10c	Merge pull request #1346 from i7an/handle-parentheses Handle parentheses in variables in commands	2025-04-22 12:02:32 +01:00
Donal McBreen	54b2c79f08	Merge pull request #1520 from basecamp/inherit-lock Inherit locks	2025-04-22 09:31:35 +01:00
Donal McBreen	d464707c32	Merge pull request #1518 from basecamp/dependabot/bundler/bundler-8bcbabbe88 Bump nokogiri from 1.18.4 to 1.18.8 in the bundler group across 1 directory	2025-04-22 09:01:50 +01:00
Donal McBreen	f5ff612846	Merge pull request #1519 from basecamp/escape-audit-line Escape the audit line	2025-04-22 09:00:49 +01:00
Donal McBreen	04568dea2f	Inherit locks We'll set the KAMAL_LOCK environment when calling run hooks. If set to true we have the lock and the hook will not need to acquire it again if it runs kamal commands. Fixes: https://github.com/basecamp/kamal/issues/1517	2025-04-22 09:00:22 +01:00
Donal McBreen	63f65d60c6	Escape the audit line Makes it compatible with zsh.	2025-04-22 08:26:51 +01:00
dependabot[bot]	5145289625	Bump nokogiri in the bundler group across 1 directory Bumps the bundler group with 1 update in the / directory: [nokogiri](https://github.com/sparklemotion/nokogiri). Updates `nokogiri` from 1.18.4 to 1.18.8 - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.18.4...v1.18.8) --- updated-dependencies: - dependency-name: nokogiri dependency-version: 1.18.8 dependency-type: indirect dependency-group: bundler ... Signed-off-by: dependabot[bot] <support@github.com>	2025-04-22 04:53:20 +00:00
Donal McBreen	aa57462c1b	Merge pull request #1470 from pdl/chore/initialise-gh-status-checks-inside-begin chore: put github status checks inside block	2025-04-21 16:27:08 +01:00
Donal McBreen	d0c9af20d8	Merge pull request #1405 from mike-weiner/fix-1399-fail-exec-without-cmd Raise an error to the user if the exec command parsed is blank	2025-04-21 16:16:06 +01:00
Donal McBreen	f898fb8cb7	Merge pull request #1515 from basecamp/build-green-output Redirect buildx build output to stdout	2025-04-21 10:48:22 +01:00
Donal McBreen	400fbcea1f	Merge pull request #1514 from basecamp/hook-kamal-roles Add KAMAL_ROLES to hook env variables	2025-04-21 10:22:57 +01:00
Donal McBreen	93d1bd1369	Redirect buildx build output to stdout Docker buildx build outputs the build logs to stderr by default. SSHKit displays stderr logs in red, which can suggest that an error has occurred. Redirect the output to stdout, so it shows in green. If there is an error, the output will be repeated in red anyway. Fixes: https://github.com/basecamp/kamal/issues/1356	2025-04-21 10:19:36 +01:00
Donal McBreen	f768fab481	Add KAMAL_ROLES to hook env variables And add an integration test to check the env vars are set.	2025-04-21 09:45:23 +01:00
Donal McBreen	02c3b947c3	Merge pull request #1511 from basecamp/docker-login-before-exec Docker login if exec might pull image	2025-04-18 15:44:22 +01:00
Donal McBreen	7a63cacb09	Docker login if exec might pull image The `app exec` and `accessory exec` commands will run `docker run` if they are not set to reuse existing containers. This might need to pull an image so let's make sure we are logged in before running the command. Fixes: https://github.com/basecamp/kamal/issues/1163	2025-04-18 14:47:59 +01:00
Donal McBreen	cd9d01b016	Merge pull request #1510 from basecamp/pre-connect-exec-commands Run pre-connect hooks before ssh commands	2025-04-18 14:46:56 +01:00
Donal McBreen	48f5eeff09	Merge pull request #1509 from basecamp/disallow-proxy-boolean-at-root Don't allow booleans for root proxy config	2025-04-18 14:46:44 +01:00
Donal McBreen	bf64d9a0f5	Run pre-connect hooks before ssh commands We hook into the SSHKit `on` method to run the pre-connect hook before the first SSH command. This doesn't work for interactive exec commands where ssh is called directly. Fixes: https://github.com/basecamp/kamal/issues/1157	2025-04-18 14:30:52 +01:00
Donal McBreen	8d5ed62d30	Don't allow booleans for root proxy config Setting it to a false or true doesn't affect the config so shouldn't be allowed. true/false are for role level configurations. Fixes: https://github.com/basecamp/kamal/issues/1120	2025-04-18 14:25:29 +01:00
Donal McBreen	58d5c7fb15	Merge pull request #1508 from basecamp/app-hosts Add KAMAL.app_hosts	2025-04-18 13:26:10 +01:00
Donal McBreen	e4e39c31e3	Add KAMAL.app_hosts KAMAL.hosts includes accessory and apps hosts. Add KAMAL.app_hosts which does not include accessory only hosts and use it for app specific commands. Fixes: - https://github.com/basecamp/kamal/issues/1059 - https://github.com/basecamp/kamal/issues/1148	2025-04-18 13:15:50 +01:00
Donal McBreen	5c71f2ba5a	Merge pull request #1507 from basecamp/fix-accessory-setup Fix accessory setup	2025-04-18 11:16:52 +01:00
Donal McBreen	05f04f4c10	Merge pull request #1506 from basecamp/registry-login-on-push-and-pull Move docker login into build command	2025-04-18 10:56:14 +01:00
Donal McBreen	03cac7ae3d	Skip existing containers on accessory boot When booting an accessory, check for the container first and skip boot if it exists. This allows us to rerun `kamal setup` on hosts with accessories without raising an error. Fixes: https://github.com/basecamp/kamal/issues/488	2025-04-18 10:53:26 +01:00
Donal McBreen	399f1526af	Handle role filter when booting accessories Filter the accessory hosts via KAMAL.accessory_hosts, which correctly handles role and host filters. Fixes: https://github.com/basecamp/kamal/issues/935	2025-04-18 10:20:54 +01:00
Donal McBreen	84fa30e376	Merge pull request #1501 from basecamp/accessory-only Allow accessory only configurations	2025-04-18 09:58:38 +01:00
Donal McBreen	098c937bab	Move docker login into build command We only need to run the docker login commands for pushing and pulling images. So let's move the logins into those commands. This ensures we are logged in when calling `kamal build` commands directly. Fixes: https://github.com/basecamp/kamal/issues/919	2025-04-18 09:57:02 +01:00
Donal McBreen	95e3edc32b	Merge pull request #1503 from basecamp/disallow-options-restart Ensure that the restart policy is unless-stopped	2025-04-17 15:16:54 +01:00
Donal McBreen	ac719dc271	Merge pull request #1502 from basecamp/remove-next-container-special-case Rely on semver for version checks	2025-04-17 15:16:00 +01:00
Donal McBreen	91f01ece1b	Ensure that the restart policy is unless-stopped No other restart policy makes sense to don't let it be changed. Fixes: https://github.com/basecamp/kamal/issues/749	2025-04-17 13:36:26 +01:00
Donal McBreen	521425c386	Rely on semver for version checks	2025-04-17 13:18:46 +01:00
Donal McBreen	55ec6ca0a6	Allow accessory only configurations If there are accessories defined in the configuration, we'll not require servers to be defined as well. This allows for accessory-only configurations which allows you to run external images with kamal-proxy for zero-downtime deployments. We don't manage image cleanup for accessories though so the user will need to deal with that themselves.	2025-04-17 11:40:03 +01:00
Donal McBreen	2a8d561094	Merge pull request #1497 from basecamp/maintenance-mode Maintenance mode	2025-04-17 09:25:58 +01:00
Donal McBreen	354530f3b8	Maintenance mode Adds support for maintenance mode to Kamal. There are two new commands: - `kamal app maintenance` - puts the app in maintenance mode - `kamal app live` - puts the app back in live mode In maintenance mode, the kamal proxy will respond to requests with a 503 status code. It will use an error page built into kamal proxy. You can use your own error page by setting `error_pages_path` in the configuration. This will copy any 4xx.html or 5xx.html files from that page to a volume mounted into the proxy container.	2025-04-17 09:11:21 +01:00
Donal McBreen	26b6c072f3	Add a writable proxy volume Maps in and external /home/kamal-proxy/.app-config volume that we can use to map files to the proxy. Can be used to store custom maintenance pages or SSL certificates.	2025-04-17 09:08:36 +01:00
Donal McBreen	3c1fbb41cb	Merge pull request #1499 from basecamp/custom-proxy-image Custom proxy image	2025-04-17 09:05:27 +01:00
Donal McBreen	8ceeda6ac9	Extract proxy_default_boot_options	2025-04-17 08:47:01 +01:00
Donal McBreen	dd9048e09c	Allow version 'next'	2025-04-17 08:20:25 +01:00
Donal McBreen	bd81632439	Set DEBUG for integration test output	2025-04-16 16:54:46 +01:00
Donal McBreen	85320dbc51	Custom proxy image registry, repo and version Use the --registry, --repository and --image_version options of `kamal proxy boot_config set` to change the kamal-proxy image used. We'll still insist that the image version is at least as high as the minimum.	2025-04-16 16:54:46 +01:00
Donal McBreen	c9a755bde6	Extract echo_boot_config/docker_run methods	2025-04-16 14:33:15 +01:00
Donal McBreen	c3a9a3c1eb	Merge pull request #1494 from basecamp/integration-test-registry-3 Use registry:3 image for the integration tests	2025-04-15 11:36:01 +01:00
Donal McBreen	215fd2faed	Use registry:3 image for the integration tests v3 was recently released which broke the integration tests. Update them to use the correct config file. Set the major version to prevent this from happening when v4 is released.	2025-04-15 10:55:36 +01:00
Donal McBreen	ec28caa83f	Merge pull request #1476 from aliismayilov/aws-secrets-json Enforce JSON output format for aws secrets manager command	2025-04-15 10:05:31 +01:00
Donal McBreen	a71ea08fb6	Merge pull request #1464 from basecamp/dependabot/bundler/bundler-bee64d1bf2 Bump nokogiri from 1.18.3 to 1.18.4 in the bundler group across 1 directory	2025-04-15 09:46:50 +01:00
Ali Ismayilov	0b28a54518	Enforce JSON output format for aws command	2025-04-02 18:18:03 +02:00
Daniel Perrett	d7dbef1c9e	chore: guard GithubStatusChecks.new, which makes calls on initialize	2025-03-25 14:03:22 +00:00
Donal McBreen	8fe2f92164	Merge pull request #1466 from basecamp/env-secrets-tidy Tidy up the env secrets handling	2025-03-24 09:38:22 +00:00
Donal McBreen	fb95b38e73	Tidy up the env secrets handling The secrets accessor was only used in the tests so remove it. Skip the memoization, it makes things slightly harder to follow and it's not needed.	2025-03-24 09:21:27 +00:00
Donal McBreen	3aef9303c3	Merge pull request #1465 from visini/feature/aliased-secrets-within-tags Add ability to alias secrets for tags	2025-03-24 09:15:27 +00:00
Camillo Visini	c1d8ce7f70	Add ability to alias secrets for tags Aliasing for secrets was introduced in #1439, but only supported "top-level" secrets. This adds support for aliasing/mapping secrets for tags.	2025-03-22 12:11:48 +01:00
dependabot[bot]	eeb5c01fc5	Bump nokogiri in the bundler group across 1 directory Bumps the bundler group with 1 update in the / directory: [nokogiri](https://github.com/sparklemotion/nokogiri). Updates `nokogiri` from 1.18.3 to 1.18.4 - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/main/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.18.3...v1.18.4) --- updated-dependencies: - dependency-name: nokogiri dependency-type: indirect dependency-group: bundler ... Signed-off-by: dependabot[bot] <support@github.com>	2025-03-21 23:59:58 +00:00
Donal McBreen	58e23f9167	Merge pull request #1459 from basecamp/xargs-proxy-options Use xargs to handle spaces in proxy options	2025-03-19 09:19:12 +00:00
Donal McBreen	7fa27faaca	Use xargs to handle spaces in proxy options We cat the options file, append the proxy image and then pass it to xargs to ensure it handles spaces correctly. Works better than using eval which can handle spaces but tries to evaluate things like backticks. Fixes: https://github.com/basecamp/kamal/issues/1448	2025-03-18 08:46:31 +00:00
Donal McBreen	a02826284d	Merge pull request #1454 from basecamp/proxy-disable-redirect Add `ssl_redirect` option	2025-03-18 08:46:00 +00:00
Donal McBreen	4d78afaf1b	Merge pull request #1452 from basecamp/dependabot/bundler/bundler-9e375bb185 Bump the bundler group across 1 directory with 2 updates	2025-03-17 14:44:59 +00:00
Kevin McConnell	d4ab010b01	Add `ssl_redirect` proxy option	2025-03-12 11:11:14 +00:00
Kevin McConnell	3c9a3f2264	Require proxy version v0.8.7	2025-03-12 10:50:45 +00:00
dependabot[bot]	8098ed1fd1	Bump the bundler group across 1 directory with 2 updates Bumps the bundler group with 2 updates in the / directory: [rack](https://github.com/rack/rack) and [uri](https://github.com/ruby/uri). Updates `rack` from 3.1.10 to 3.1.12 - [Release notes](https://github.com/rack/rack/releases) - [Changelog](https://github.com/rack/rack/blob/main/CHANGELOG.md) - [Commits](https://github.com/rack/rack/compare/v3.1.10...v3.1.12) Updates `uri` from 1.0.2 to 1.0.3 - [Release notes](https://github.com/ruby/uri/releases) - [Commits](https://github.com/ruby/uri/compare/v1.0.2...v1.0.3) --- updated-dependencies: - dependency-name: rack dependency-type: indirect dependency-group: bundler - dependency-name: uri dependency-type: indirect dependency-group: bundler ... Signed-off-by: dependabot[bot] <support@github.com>	2025-03-10 22:29:34 +00:00
Donal McBreen	0d034ec5dc	Merge pull request #1439 from matthewbjones/feature/aliased-secrets Adds the ability to alias/map secrets	2025-03-10 10:06:45 +00:00
Donal McBreen	598bd65b78	Merge pull request #1449 from basecamp/kamal-proxy-drain-timeout-fix Update to kamal-proxy 0.8.6	2025-03-10 10:06:14 +00:00
Donal McBreen	36f4e90a76	Update to kamal-proxy 0.8.6 Includes a fix for locking the proxy while draining	2025-03-07 11:33:17 +00:00
Matthew Jones	973fa1a7ff	Adds the ability to alias/map secrets	2025-03-04 07:23:26 -07:00
Michael Weiner	5e87b6d58e	Use double-quotes on UT	2025-03-04 06:32:08 -06:00
Michael Weiner	5377871278	Add UT for missing command	2025-02-08 13:09:57 -06:00
Michael Weiner	91259720b2	Fail kamal app exec without a CMD	2025-02-07 20:13:18 -06:00
Ivan Yurchanka	7627f74e45	Handle parentheses in variables in commands	2025-01-08 17:13:10 +01:00