Merge branch 'basecamp:main' into buildpacks

Gemfile.lock

@@ -82,15 +82,15 @@ GEM
     net-sftp (4.0.0)
       net-ssh (>= 5.0.0, < 8.0.0)
     net-ssh (7.3.0)
-    nokogiri (1.18.3-aarch64-linux-musl)
+    nokogiri (1.18.4-aarch64-linux-musl)
       racc (~> 1.4)
-    nokogiri (1.18.3-arm64-darwin)
+    nokogiri (1.18.4-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.18.3-x86_64-darwin)
+    nokogiri (1.18.4-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.18.3-x86_64-linux-gnu)
+    nokogiri (1.18.4-x86_64-linux-gnu)
       racc (~> 1.4)
-    nokogiri (1.18.3-x86_64-linux-musl)
+    nokogiri (1.18.4-x86_64-linux-musl)
       racc (~> 1.4)
     ostruct (0.6.1)
     parallel (1.26.3)

lib/kamal/cli/accessory.rb

@@ -1,4 +1,5 @@
 require "active_support/core_ext/array/conversions"
+require "concurrent/array"
 
 class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "boot [NAME]", "Boot new accessory service on host (use NAME=all to boot all accessories)"
@@ -10,6 +11,16 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
         prepare(name) if prepare
 
         with_accessory(name) do |accessory, hosts|
+          booted_hosts = Concurrent::Array.new
+          on(hosts) do |host|
+            booted_hosts << host.to_s if capture_with_info(*accessory.info(all: true, quiet: true)).strip.presence
+          end
+
+          if booted_hosts.any?
+            say "Skipping booting `#{name}` on #{booted_hosts.sort.join(", ")}, a container already exists", :yellow
+            hosts -= booted_hosts
+          end
+
           directories(name)
           upload(name)
 
@@ -130,6 +141,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
   option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
   def exec(name, *cmd)
+    pre_connect_if_required
+
     cmd = Kamal::Utils.join_commands(cmd)
     with_accessory(name) do |accessory, hosts|
       case
@@ -139,6 +152,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
       when options[:interactive]
         say "Launching interactive command via SSH from new container...", :magenta
+        on(accessory.hosts.first) { execute *KAMAL.registry.login }
         run_locally { exec accessory.execute_in_new_container_over_ssh(cmd) }
 
       when options[:reuse]
@@ -151,6 +165,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
       else
         say "Launching command from new container...", :magenta
         on(hosts) do |host|
+          execute *KAMAL.registry.login
           execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
           puts_by_host host, capture_with_info(*accessory.execute_in_new_container(cmd))
         end
@@ -275,11 +290,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     end
 
     def accessory_hosts(accessory)
-      if KAMAL.specific_hosts&.any?
+      KAMAL.accessory_hosts & accessory.hosts
-        KAMAL.specific_hosts & accessory.hosts
-      else
-        accessory.hosts
-      end
     end
 
     def remove_accessory(name)

lib/kamal/cli/app.rb

@@ -7,9 +7,11 @@ class Kamal::Cli::App < Kamal::Cli::Base
         say "Start container with version #{version} (or reboot if already running)...", :magenta
 
         # Assets are prepared in a separate step to ensure they are on all hosts before booting
-        on(KAMAL.hosts) do
+        on(KAMAL.app_hosts) do
+          Kamal::Cli::App::ErrorPages.new(host, self).run
+
           KAMAL.roles_on(host).each do |role|
-            Kamal::Cli::App::PrepareAssets.new(host, role, self).run
+            Kamal::Cli::App::Assets.new(host, role, self).run
           end
         end
 
@@ -31,7 +33,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
         end
 
         # Tag once the app booted on all hosts
-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
           execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
           execute *KAMAL.app.tag_latest_image
         end
@@ -42,7 +44,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "start", "Start existing app container on servers"
   def start
     with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
@@ -65,7 +67,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "stop", "Stop app container on servers"
   def stop
     with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
@@ -89,7 +91,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
   # FIXME: Drop in favor of just containers?
   desc "details", "Show details about app containers"
   def details
-    on(KAMAL.hosts) do |host|
+    on(KAMAL.app_hosts) do |host|
       roles = KAMAL.roles_on(host)
 
       roles.each do |role|
@@ -104,6 +106,8 @@ class Kamal::Cli::App < Kamal::Cli::Base
   option :env, aliases: "-e", type: :hash, desc: "Set environment variables for the command"
   option :detach, type: :boolean, default: false, desc: "Execute command in a detached container"
   def exec(*cmd)
+    pre_connect_if_required
+
     if (incompatible_options = [ :interactive, :reuse ].select { |key| options[:detach] && options[key] }.presence)
       raise ArgumentError, "Detach is not compatible with #{incompatible_options.join(" or ")}"
     end
@@ -123,6 +127,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
       say "Get most recent version available as an image...", :magenta unless options[:version]
       using_version(version_or_latest) do |version|
         say "Launching interactive command with version #{version} via SSH from new container on #{KAMAL.primary_host}...", :magenta
+        on(KAMAL.primary_host) { execute *KAMAL.registry.login }
         run_locally do
           exec KAMAL.app(role: KAMAL.primary_role, host: KAMAL.primary_host).execute_in_new_container_over_ssh(cmd, env: env)
         end
@@ -133,7 +138,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
       using_version(options[:version] || current_running_version) do |version|
         say "Launching command with version #{version} from existing container...", :magenta
 
-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
           roles = KAMAL.roles_on(host)
 
           roles.each do |role|
@@ -147,7 +152,9 @@ class Kamal::Cli::App < Kamal::Cli::Base
       say "Get most recent version available as an image...", :magenta unless options[:version]
       using_version(version_or_latest) do |version|
         say "Launching command with version #{version} from new container...", :magenta
-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
+          execute *KAMAL.registry.login
+
           roles = KAMAL.roles_on(host)
 
           roles.each do |role|
@@ -161,7 +168,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "containers", "Show app containers on servers"
   def containers
-    on(KAMAL.hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_containers) }
+    on(KAMAL.app_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_containers) }
   end
 
   desc "stale_containers", "Detect app stale containers"
@@ -170,7 +177,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
     stop = options[:stop]
 
     with_lock_if_stopping do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
@@ -193,7 +200,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "images", "Show app images on servers"
   def images
-    on(KAMAL.hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_images) }
+    on(KAMAL.app_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_images) }
   end
 
   desc "logs", "Show log lines from app on servers (use --help to show options)"
@@ -229,7 +236,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
     else
       lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
@@ -249,14 +256,44 @@ class Kamal::Cli::App < Kamal::Cli::Base
       stop
       remove_containers
       remove_images
-      remove_app_directory
+      remove_app_directories
+    end
+  end
+
+  desc "live", "Set the app to live mode"
+  def live
+    with_lock do
+      on(KAMAL.proxy_hosts) do |host|
+        roles = KAMAL.roles_on(host)
+
+        roles.each do |role|
+          execute *KAMAL.app(role: role, host: host).live if role.running_proxy?
+        end
+      end
+    end
+  end
+
+  desc "maintenance", "Set the app to maintenance mode"
+  option :drain_timeout, type: :numeric, desc: "How long to allow in-flight requests to complete (defaults to drain_timeout from config)"
+  option :message, type: :string, desc: "Message to display to clients while stopped"
+  def maintenance
+    maintenance_options = { drain_timeout: options[:drain_timeout] || KAMAL.config.drain_timeout, message: options[:message] }
+
+    with_lock do
+      on(KAMAL.proxy_hosts) do |host|
+        roles = KAMAL.roles_on(host)
+
+        roles.each do |role|
+          execute *KAMAL.app(role: role, host: host).maintenance(**maintenance_options) if role.running_proxy?
+        end
+      end
     end
   end
 
   desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true
   def remove_container(version)
     with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
@@ -270,7 +307,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "remove_containers", "Remove all app containers from servers", hide: true
   def remove_containers
     with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
@@ -284,30 +321,33 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "remove_images", "Remove all app images from servers", hide: true
   def remove_images
     with_lock do
-      on(KAMAL.hosts) do
+      on(KAMAL.app_hosts) do
         execute *KAMAL.auditor.record("Removed all app images"), verbosity: :debug
         execute *KAMAL.app.remove_images
       end
     end
   end
 
-  desc "remove_app_directory", "Remove the service directory from servers", hide: true
+  desc "remove_app_directories", "Remove the app directories from servers", hide: true
-  def remove_app_directory
+  def remove_app_directories
     with_lock do
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
-          execute *KAMAL.auditor.record("Removed #{KAMAL.config.app_directory} on all servers", role: role), verbosity: :debug
+          execute *KAMAL.auditor.record("Removed #{KAMAL.config.app_directory}", role: role), verbosity: :debug
           execute *KAMAL.server.remove_app_directory, raise_on_non_zero_exit: false
         end
+
+        execute *KAMAL.auditor.record("Removed #{KAMAL.config.app_directory}"), verbosity: :debug
+        execute *KAMAL.app.remove_proxy_app_directory, raise_on_non_zero_exit: false
       end
     end
   end
 
   desc "version", "Show app version currently running on servers"
   def version
-    on(KAMAL.hosts) do |host|
+    on(KAMAL.app_hosts) do |host|
       role = KAMAL.roles_on(host).first
       puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).current_running_version).strip
     end
@@ -350,6 +390,6 @@ class Kamal::Cli::App < Kamal::Cli::Base
     end
 
     def host_boot_groups
-      KAMAL.config.boot.limit ? KAMAL.hosts.each_slice(KAMAL.config.boot.limit).to_a : [ KAMAL.hosts ]
+      KAMAL.config.boot.limit ? KAMAL.app_hosts.each_slice(KAMAL.config.boot.limit).to_a : [ KAMAL.app_hosts ]
     end
 end

lib/kamal/cli/app/assets.rb

@@ -1,4 +1,4 @@
-class Kamal::Cli::App::PrepareAssets
+class Kamal::Cli::App::Assets
   attr_reader :host, :role, :sshkit
   delegate :execute, :capture_with_info, :info, to: :sshkit
   delegate :assets?, to: :role

lib/kamal/cli/app/boot.rb

@@ -70,6 +70,7 @@ class Kamal::Cli::App::Boot
     def stop_old_version(version)
       execute *app.stop(version: version), raise_on_non_zero_exit: false
       execute *app.clean_up_assets if assets?
+      execute *app.clean_up_error_pages if KAMAL.config.error_pages_path
     end
 
     def release_barrier

lib/kamal/cli/app/error_pages.rb

@@ -0,0 +1,33 @@
+class Kamal::Cli::App::ErrorPages
+  ERROR_PAGES_GLOB = "{4??.html,5??.html}"
+
+  attr_reader :host, :sshkit
+  delegate :upload!, :execute, to: :sshkit
+
+  def initialize(host, sshkit)
+    @host = host
+    @sshkit = sshkit
+  end
+
+  def run
+    if KAMAL.config.error_pages_path
+      with_error_pages_tmpdir do |local_error_pages_dir|
+        execute *KAMAL.app.create_error_pages_directory
+        upload! local_error_pages_dir, KAMAL.config.proxy_error_pages_directory, mode: "0700", recursive: true
+      end
+    end
+  end
+
+  private
+    def with_error_pages_tmpdir
+      Dir.mktmpdir("kamal-error-pages") do |tmpdir|
+        error_pages_dir = File.join(tmpdir, KAMAL.config.version)
+        FileUtils.mkdir(error_pages_dir)
+
+        if (files = Dir[File.join(KAMAL.config.error_pages_path, ERROR_PAGES_GLOB)]).any?
+          FileUtils.cp(files, error_pages_dir)
+          yield error_pages_dir
+        end
+      end
+    end
+end

lib/kamal/cli/base.rb

@@ -147,12 +147,16 @@ module Kamal::Cli
       end
 
       def on(*args, &block)
+        pre_connect_if_required
+
+        super
+      end
+
+      def pre_connect_if_required
         if !KAMAL.connected?
           run_hook "pre-connect"
           KAMAL.connected = true
         end
-
-        super
       end
 
       def command

lib/kamal/cli/build.rb

@@ -15,6 +15,8 @@ class Kamal::Cli::Build < Kamal::Cli::Base
     cli = self
 
     ensure_docker_installed
+    login_to_registry_locally
+
     run_hook "pre-build"
 
     uncommitted_changes = Kamal::Git.uncommitted_changes
@@ -61,14 +63,16 @@ class Kamal::Cli::Build < Kamal::Cli::Base
 
   desc "pull", "Pull app image from registry onto servers"
   def pull
+    login_to_registry_remotely
+
     if (first_hosts = mirror_hosts).any?
       #  Pull on a single host per mirror first to seed them
       say "Pulling image on #{first_hosts.join(", ")} to seed the #{"mirror".pluralize(first_hosts.count)}...", :magenta
       pull_on_hosts(first_hosts)
       say "Pulling image on remaining hosts...", :magenta
-      pull_on_hosts(KAMAL.hosts - first_hosts)
+      pull_on_hosts(KAMAL.app_hosts - first_hosts)
     else
-      pull_on_hosts(KAMAL.hosts)
+      pull_on_hosts(KAMAL.app_hosts)
     end
   end
 
@@ -159,9 +163,9 @@ class Kamal::Cli::Build < Kamal::Cli::Base
     end
 
     def mirror_hosts
-      if KAMAL.hosts.many?
+      if KAMAL.app_hosts.many?
         mirror_hosts = Concurrent::Hash.new
-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
           first_mirror = capture_with_info(*KAMAL.builder.first_mirror).strip.presence
           mirror_hosts[first_mirror] ||= host.to_s if first_mirror
         rescue SSHKit::Command::Failed => e
@@ -181,4 +185,16 @@ class Kamal::Cli::Build < Kamal::Cli::Base
         execute *KAMAL.builder.validate_image
       end
     end
+
+    def login_to_registry_locally
+      run_locally do
+        execute *KAMAL.registry.login
+      end
+    end
+
+    def login_to_registry_remotely
+      on(KAMAL.app_hosts) do
+        execute *KAMAL.registry.login
+      end
+    end
 end

lib/kamal/cli/main.rb

@@ -20,9 +20,6 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     runtime = print_runtime do
       invoke_options = deploy_options
 
-      say "Log into image registry...", :magenta
-      invoke "kamal:cli:registry:login", [], invoke_options.merge(skip_local: options[:skip_push])
-
       if options[:skip_push]
         say "Pull app image...", :magenta
         invoke "kamal:cli:build:pull", [], invoke_options
@@ -52,7 +49,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s
   end
 
-  desc "redeploy", "Deploy app to servers without bootstrapping servers, starting kamal-proxy, pruning, and registry login"
+  desc "redeploy", "Deploy app to servers without bootstrapping servers, starting kamal-proxy and pruning"
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def redeploy
     runtime = print_runtime do
@@ -197,10 +194,10 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     confirming "This will replace Traefik with kamal-proxy and restart all accessories" do
       with_lock do
         if options[:rolling]
-          (KAMAL.hosts | KAMAL.accessory_hosts).each do |host|
+          KAMAL.hosts.each do |host|
             KAMAL.with_specific_hosts(host) do
               say "Upgrading #{host}...", :magenta
-              if KAMAL.hosts.include?(host)
+              if KAMAL.app_hosts.include?(host)
                 invoke "kamal:cli:proxy:upgrade", [], options.merge(confirmed: true, rolling: false)
                 reset_invocation(Kamal::Cli::Proxy)
               end
@@ -256,7 +253,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   private
     def container_available?(version)
       begin
-        on(KAMAL.hosts) do
+        on(KAMAL.app_hosts) do
           KAMAL.roles_on(host).each do |role|
             container_id = capture_with_info(*KAMAL.app(role: role, host: host).container_id_for_version(version))
             raise "Container not found" unless container_id.present?

lib/kamal/cli/proxy.rb

@@ -27,6 +27,9 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
   option :http_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTP_PORT, desc: "HTTP port to publish on the host"
   option :https_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTPS_PORT, desc: "HTTPS port to publish on the host"
   option :log_max_size, type: :string, default: Kamal::Configuration::PROXY_LOG_MAX_SIZE, desc: "Max size of proxy logs"
+  option :registry, type: :string, default: nil, desc: "Registry to use for the proxy image"
+  option :repository, type: :string, default: nil, desc: "Repository for the proxy image"
+  option :image_version, type: :string, default: nil, desc: "Version of the proxy to run"
   option :docker_options, type: :array, default: [], desc: "Docker options to pass to the proxy container", banner: "option=value option2=value2"
   def boot_config(subcommand)
     case subcommand
@@ -37,17 +40,43 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
         *options[:docker_options].map { |option| "--#{option}" }
       ]
 
+      image = [
+        options[:registry].presence,
+        options[:repository].presence || KAMAL.config.proxy_repository_name,
+        KAMAL.config.proxy_image_name
+      ].compact.join("/")
+
+      image_version = options[:image_version]
+
       on(KAMAL.proxy_hosts) do |host|
         execute(*KAMAL.proxy.ensure_proxy_directory)
-        upload! StringIO.new(boot_options.join(" ")), KAMAL.config.proxy_options_file
+        if boot_options != KAMAL.config.proxy_default_boot_options
+          upload! StringIO.new(boot_options.join(" ")), KAMAL.config.proxy_options_file
+        else
+          execute *KAMAL.proxy.reset_boot_options, raise_on_non_zero_exit: false
+        end
+
+        if image != KAMAL.config.proxy_image_default
+          upload! StringIO.new(image), KAMAL.config.proxy_image_file
+        else
+          execute *KAMAL.proxy.reset_image, raise_on_non_zero_exit: false
+        end
+
+        if image_version
+          upload! StringIO.new(image_version), KAMAL.config.proxy_image_version_file
+        else
+          execute *KAMAL.proxy.reset_image_version, raise_on_non_zero_exit: false
+        end
       end
     when "get"
       on(KAMAL.proxy_hosts) do |host|
-        puts "Host #{host}: #{capture_with_info(*KAMAL.proxy.get_boot_options)}"
+        puts "Host #{host}: #{capture_with_info(*KAMAL.proxy.boot_config)}"
       end
     when "reset"
       on(KAMAL.proxy_hosts) do |host|
-        execute *KAMAL.proxy.reset_boot_options
+        execute *KAMAL.proxy.reset_boot_options, raise_on_non_zero_exit: false
+        execute *KAMAL.proxy.reset_image, raise_on_non_zero_exit: false
+        execute *KAMAL.proxy.reset_image_version, raise_on_non_zero_exit: false
       end
     else
       raise ArgumentError, "Unknown boot_config subcommand #{subcommand}"

lib/kamal/cli/server.rb

@@ -2,8 +2,10 @@ class Kamal::Cli::Server < Kamal::Cli::Base
   desc "exec", "Run a custom command on the server (use --help to show options)"
   option :interactive, type: :boolean, aliases: "-i", default: false, desc: "Run the command interactively (use for console/bash)"
   def exec(*cmd)
+    pre_connect_if_required
+
     cmd = Kamal::Utils.join_commands(cmd)
-    hosts = KAMAL.hosts | KAMAL.accessory_hosts
+    hosts = KAMAL.hosts
 
     case
     when options[:interactive]
@@ -27,7 +29,7 @@ class Kamal::Cli::Server < Kamal::Cli::Base
     with_lock do
       missing = []
 
-      on(KAMAL.hosts | KAMAL.accessory_hosts) do |host|
+      on(KAMAL.hosts) do |host|
         unless execute(*KAMAL.docker.installed?, raise_on_non_zero_exit: false)
           if execute(*KAMAL.docker.superuser?, raise_on_non_zero_exit: false)
             info "Missing Docker on #{host}. Installing…"

lib/kamal/commander.rb

@@ -5,7 +5,7 @@ require "active_support/core_ext/object/blank"
 class Kamal::Commander
   attr_accessor :verbosity, :holding_lock, :connected
   attr_reader :specific_roles, :specific_hosts
-  delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :proxy_hosts, :accessory_hosts, to: :specifics
+  delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :app_hosts, :proxy_hosts, :accessory_hosts, to: :specifics
 
   def initialize
     reset

lib/kamal/commander/specifics.rb

@@ -18,6 +18,10 @@ class Kamal::Commander::Specifics
     roles.select { |role| role.hosts.include?(host.to_s) }
   end
 
+  def app_hosts
+    config.app_hosts & specified_hosts
+  end
+
   def proxy_hosts
     config.proxy_hosts & specified_hosts
   end

lib/kamal/commands/accessory.rb

@@ -37,8 +37,8 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     docker :container, :stop, service_name
   end
 
-  def info
+  def info(all: false, quiet: false)
-    docker :ps, *service_filter
+    docker :ps, *("-a" if all), *("-q" if quiet), *service_filter
   end
 
   def logs(timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil)

lib/kamal/commands/app.rb

@@ -1,5 +1,5 @@
 class Kamal::Commands::App < Kamal::Commands::Base
-  include Assets, Containers, Execution, Images, Logging, Proxy
+  include Assets, Containers, ErrorPages, Execution, Images, Logging, Proxy
 
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 

lib/kamal/commands/app/error_pages.rb

@@ -0,0 +1,9 @@
+module Kamal::Commands::App::ErrorPages
+  def create_error_pages_directory
+    make_directory(config.proxy_error_pages_directory)
+  end
+
+  def clean_up_error_pages
+    [ :find, config.proxy_error_pages_directory, "-mindepth", "1", "-maxdepth", "1", "!", "-name", KAMAL.config.version, "-exec", "rm", "-rf", "{} +" ]
+  end
+end

lib/kamal/commands/app/proxy.rb

@@ -9,6 +9,18 @@ module Kamal::Commands::App::Proxy
     proxy_exec :remove, role.container_prefix
   end
 
+  def live
+    proxy_exec :resume, role.container_prefix
+  end
+
+  def maintenance(**options)
+    proxy_exec :stop, role.container_prefix, *role.proxy.stop_command_args(**options)
+  end
+
+  def remove_proxy_app_directory
+    remove_directory config.proxy_app_directory
+  end
+
   private
     def proxy_exec(*command)
       docker :exec, proxy_container_name, "kamal-proxy", *command

lib/kamal/commands/base.rb

@@ -68,6 +68,10 @@ module Kamal::Commands
         combine *commands, by: "||"
       end
 
+      def substitute(*commands)
+        "\$\(#{commands.join(" ")}\)"
+      end
+
       def xargs(command)
         [ :xargs, command ].flatten
       end

lib/kamal/commands/proxy.rb

@@ -2,14 +2,7 @@ class Kamal::Commands::Proxy < Kamal::Commands::Base
   delegate :argumentize, :optionize, to: Kamal::Utils
 
   def run
-    pipe \
+    pipe boot_config, xargs(docker_run)
-      [ :echo, "\$\(#{get_boot_options.join(" ")}\) #{config.proxy_image}" ],
-      xargs(docker(:run,
-        "--name", container_name,
-        "--network", "kamal",
-        "--detach",
-        "--restart", "unless-stopped",
-        "--volume", "kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy"))
   end
 
   def start
@@ -31,7 +24,7 @@ class Kamal::Commands::Proxy < Kamal::Commands::Base
   def version
     pipe \
       docker(:inspect, container_name, "--format '{{.Config.Image}}'"),
-      [ :cut, "-d:", "-f2" ]
+      [ :awk, "-F:", "'{print \$NF}'" ]
   end
 
   def logs(timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil)
@@ -72,16 +65,51 @@ class Kamal::Commands::Proxy < Kamal::Commands::Base
     remove_directory config.proxy_directory
   end
 
-  def get_boot_options
+  def boot_config
-    combine [ :cat, config.proxy_options_file, "2>", "/dev/null" ], [ :echo, "\"#{config.proxy_options_default.join(" ")}\"" ], by: "||"
+    [ :echo, "#{substitute(read_boot_options)} #{substitute(read_image)}:#{substitute(read_image_version)}" ]
+  end
+
+  def read_boot_options
+    read_file(config.proxy_options_file, default: config.proxy_options_default.join(" "))
+  end
+
+  def read_image
+    read_file(config.proxy_image_file, default: config.proxy_image_default)
+  end
+
+  def read_image_version
+    read_file(config.proxy_image_version_file, default: Kamal::Configuration::PROXY_MINIMUM_VERSION)
   end
 
   def reset_boot_options
     remove_file config.proxy_options_file
   end
 
+  def reset_image
+    remove_file config.proxy_image_file
+  end
+
+  def reset_image_version
+    remove_file config.proxy_image_version_file
+  end
+
   private
     def container_name
       config.proxy_container_name
     end
+
+    def read_file(file, default: nil)
+      combine [ :cat, file, "2>", "/dev/null" ], [ :echo, "\"#{default}\"" ], by: "||"
+    end
+
+    def docker_run
+      docker \
+        :run,
+        "--name", container_name,
+        "--network", "kamal",
+        "--detach",
+        "--restart", "unless-stopped",
+        "--volume", "kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy",
+        *config.proxy_apps_volume.docker_args
+    end
 end

lib/kamal/configuration.rb

@@ -68,7 +68,7 @@ class Kamal::Configuration
     @env = Env.new(config: @raw_config.env || {}, secrets: secrets)
 
     @logging = Logging.new(logging_config: @raw_config.logging)
-    @proxy = Proxy.new(config: self, proxy_config: @raw_config.proxy || {})
+    @proxy = Proxy.new(config: self, proxy_config: @raw_config.key?(:proxy) ? @raw_config.proxy : {})
     @ssh = Ssh.new(config: self)
     @sshkit = Sshkit.new(config: self)
 
@@ -105,6 +105,10 @@ class Kamal::Configuration
     raw_config.minimum_version
   end
 
+  def service_and_destination
+    [ service, destination ].compact.join("-")
+  end
+
   def roles
     servers.roles
   end
@@ -121,6 +125,10 @@ class Kamal::Configuration
     (roles + accessories).flat_map(&:hosts).uniq
   end
 
+  def app_hosts
+    roles.flat_map(&:hosts).uniq
+  end
+
   def primary_host
     primary_role&.primary_host
   end
@@ -145,8 +153,12 @@ class Kamal::Configuration
     proxy_roles.flat_map(&:name)
   end
 
+  def proxy_accessories
+    accessories.select(&:running_proxy?)
+  end
+
   def proxy_hosts
-    proxy_roles.flat_map(&:hosts).uniq
+    (proxy_roles.flat_map(&:hosts) + proxy_accessories.flat_map(&:hosts)).uniq
   end
 
   def repository
@@ -210,7 +222,7 @@ class Kamal::Configuration
   end
 
   def app_directory
-    File.join apps_directory, [ service, destination ].compact.join("-")
+    File.join apps_directory, service_and_destination
   end
 
   def env_directory
@@ -229,6 +241,10 @@ class Kamal::Configuration
     raw_config.asset_path
   end
 
+  def error_pages_path
+    raw_config.error_pages_path
+  end
+
   def env_tags
     @env_tags ||= if (tags = raw_config.env["tags"])
       tags.collect { |name, config| Env::Tag.new(name, config: config, secrets: secrets) }
@@ -257,12 +273,27 @@ class Kamal::Configuration
     argumentize "--log-opt", "max-size=#{max_size}" if max_size.present?
   end
 
+  def proxy_default_boot_options
+    [
+      *(KAMAL.config.proxy_publish_args(Kamal::Configuration::PROXY_HTTP_PORT, Kamal::Configuration::PROXY_HTTPS_PORT, nil)),
+      *(KAMAL.config.proxy_logging_args(Kamal::Configuration::PROXY_LOG_MAX_SIZE))
+    ]
+  end
+
   def proxy_options_default
     [ *proxy_publish_args(PROXY_HTTP_PORT, PROXY_HTTPS_PORT), *proxy_logging_args(PROXY_LOG_MAX_SIZE) ]
   end
 
-  def proxy_image
+  def proxy_repository_name
-    "basecamp/kamal-proxy:#{PROXY_MINIMUM_VERSION}"
+    "basecamp"
+  end
+
+  def proxy_image_name
+    "kamal-proxy"
+  end
+
+  def proxy_image_default
+    "#{proxy_repository_name}/#{proxy_image_name}"
   end
 
   def proxy_container_name
@@ -277,6 +308,44 @@ class Kamal::Configuration
     File.join proxy_directory, "options"
   end
 
+  def proxy_image_file
+    File.join proxy_directory, "image"
+  end
+
+  def proxy_image_version_file
+    File.join proxy_directory, "image_version"
+  end
+
+  def proxy_apps_directory
+    File.join proxy_directory, "apps-config"
+  end
+
+  def proxy_apps_container_directory
+    "/home/kamal-proxy/.apps-config"
+  end
+
+  def proxy_apps_volume
+    Volume.new \
+      host_path: proxy_apps_directory,
+      container_path: proxy_apps_container_directory
+  end
+
+  def proxy_app_directory
+    File.join proxy_apps_directory, service_and_destination
+  end
+
+  def proxy_app_container_directory
+    File.join proxy_apps_container_directory, service_and_destination
+  end
+
+  def proxy_error_pages_directory
+    File.join proxy_app_directory, "error_pages"
+  end
+
+  def proxy_error_pages_container_directory
+    File.join proxy_app_container_directory, "error_pages"
+  end
+
   def to_h
     {
       roles: role_names,
@@ -306,22 +375,26 @@ class Kamal::Configuration
     end
 
     def ensure_required_keys_present
-      %i[ service image registry servers ].each do |key|
+      %i[ service image registry ].each do |key|
         raise Kamal::ConfigurationError, "Missing required configuration for #{key}" unless raw_config[key].present?
       end
 
-      unless role(primary_role_name).present?
+      if raw_config.servers.nil?
-        raise Kamal::ConfigurationError, "The primary_role #{primary_role_name} isn't defined"
+        raise Kamal::ConfigurationError, "No servers or accessories specified" unless raw_config.accessories.present?
-      end
+      else
+        unless role(primary_role_name).present?
+          raise Kamal::ConfigurationError, "The primary_role #{primary_role_name} isn't defined"
+        end
 
-      if primary_role.hosts.empty?
+        if primary_role.hosts.empty?
-        raise Kamal::ConfigurationError, "No servers specified for the #{primary_role.name} primary_role"
+          raise Kamal::ConfigurationError, "No servers specified for the #{primary_role.name} primary_role"
-      end
+        end
 
-      unless allow_empty_roles?
+        unless allow_empty_roles?
-        roles.each do |role|
+          roles.each do |role|
-          if role.hosts.empty?
+            if role.hosts.empty?
-            raise Kamal::ConfigurationError, "No servers specified for the #{role.name} role. You can ignore this with allow_empty_roles: true"
+              raise Kamal::ConfigurationError, "No servers specified for the #{role.name} role. You can ignore this with allow_empty_roles: true"
+            end
           end
         end
       end

lib/kamal/configuration/docs/configuration.yml

@@ -82,6 +82,12 @@ asset_path: /path/to/assets
 # See https://kamal-deploy.org/docs/hooks for more information:
 hooks_path: /user_home/kamal/hooks
 
+# Error pages
+#
+# A directory relative to the app root to find error pages for the proxy to serve.
+# Any files in the format 4xx.html or 5xx.html will be copied to the hosts.
+error_pages_path: public
+
 # Require destinations
 #
 # Whether deployments require a destination to be specified, defaults to `false`:

lib/kamal/configuration/proxy.rb

@@ -44,7 +44,8 @@ class Kamal::Configuration::Proxy
       "forward-headers": proxy_config.dig("forward_headers"),
       "tls-redirect": proxy_config.dig("ssl_redirect"),
       "log-request-header": proxy_config.dig("logging", "request_headers") || DEFAULT_LOG_REQUEST_HEADERS,
-      "log-response-header": proxy_config.dig("logging", "response_headers")
+      "log-response-header": proxy_config.dig("logging", "response_headers"),
+      "error-pages": error_pages
     }.compact
   end
 
@@ -52,6 +53,17 @@ class Kamal::Configuration::Proxy
     optionize ({ target: "#{target}:#{app_port}" }).merge(deploy_options), with: "="
   end
 
+  def stop_options(drain_timeout: nil, message: nil)
+    {
+      "drain-timeout": seconds_duration(drain_timeout),
+      message: message
+    }.compact
+  end
+
+  def stop_command_args(**options)
+    optionize stop_options(**options), with: "="
+  end
+
   def merge(other)
     self.class.new config: config, proxy_config: proxy_config.deep_merge(other.proxy_config)
   end
@@ -60,4 +72,8 @@ class Kamal::Configuration::Proxy
     def seconds_duration(value)
       value ? "#{value}s" : nil
     end
+
+    def error_pages
+      File.join config.proxy_error_pages_container_directory, config.version if config.error_pages_path
+    end
 end

lib/kamal/configuration/servers.rb

@@ -13,6 +13,13 @@ class Kamal::Configuration::Servers
 
   private
     def role_names
-      servers_config.is_a?(Array) ? [ "web" ] : servers_config.keys.sort
+      case servers_config
+      when Array
+        [ "web" ]
+      when NilClass
+        []
+      else
+        servers_config.keys.sort
+      end
     end
 end

lib/kamal/configuration/validator.rb

@@ -168,4 +168,10 @@ class Kamal::Configuration::Validator
       unknown_keys.reject! { |key| extension?(key) } if allow_extensions?
       unknown_keys_error unknown_keys if unknown_keys.present?
     end
+
+    def validate_docker_options!(options)
+      if options
+        error "Cannot set restart policy in docker options, unless-stopped is required" if options["restart"]
+      end
+    end
 end

lib/kamal/configuration/validator/accessory.rb

@@ -5,5 +5,7 @@ class Kamal::Configuration::Validator::Accessory < Kamal::Configuration::Validat
     if (config.keys & [ "host", "hosts", "roles" ]).size != 1
       error "specify one of `host`, `hosts` or `roles`"
     end
+
+    validate_docker_options!(config["options"])
   end
 end

lib/kamal/configuration/validator/role.rb

@@ -6,6 +6,7 @@ class Kamal::Configuration::Validator::Role < Kamal::Configuration::Validator
       validate_servers!(config)
     else
       super
+      validate_docker_options!(config["options"])
     end
   end
 end

lib/kamal/configuration/validator/servers.rb

@@ -1,6 +1,6 @@
 class Kamal::Configuration::Validator::Servers < Kamal::Configuration::Validator
   def validate!
-    validate_type! config, Array, Hash
+    validate_type! config, Array, Hash, NilClass
 
     validate_servers! config if config.is_a?(Array)
   end

lib/kamal/secrets/adapters/aws_secrets_manager.rb

@@ -26,6 +26,7 @@ class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Ba
     def get_from_secrets_manager(secrets, account: nil)
       args = [ "aws", "secretsmanager", "batch-get-secret-value", "--secret-id-list" ] + secrets.map(&:shellescape)
       args += [ "--profile", account.shellescape ] if account
+      args += [ "--output", "json" ]
       cmd = args.join(" ")
 
       `#{cmd}`.tap do |secrets|

test/cli/accessory_test.rb

@@ -115,6 +115,7 @@ class CliAccessoryTest < CliTestCase
 
   test "exec" do
     run_command("exec", "mysql", "mysql -v").tap do |output|
+      assert_match "docker login private.registry -u [REDACTED] -p [REDACTED]", output
       assert_match "Launching command from new container", output
       assert_match "mysql -v", output
     end
@@ -251,6 +252,19 @@ class CliAccessoryTest < CliTestCase
     end
   end
 
+  test "boot with web role filter" do
+    run_command("boot", "redis", "-r", "web").tap do |output|
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output
+      assert_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output
+    end
+  end
+
+  test "boot with workers role filter" do
+    run_command("boot", "redis", "-r", "workers").tap do |output|
+      assert_no_match "docker run", output
+    end
+  end
+
   private
     def run_command(*command)
       stdouted { Kamal::Cli::Accessory.start([ *command, "-c", "test/fixtures/deploy_with_accessories_with_different_registries.yml" ]) }

test/cli/app_test.rb

@@ -192,6 +192,34 @@ class CliAppTest < CliTestCase
     Thread.report_on_exception = true
   end
 
+  test "boot with only workers" do
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running").at_least_once # workers health check
+
+    run_command("boot", config: :with_only_workers, host: nil).tap do |output|
+      assert_match /First workers container is healthy on 1.1.1.\d, booting any other roles/, output
+      assert_no_match "kamal-proxy", output
+    end
+  end
+
+  test "boot with error pages" do
+    with_error_pages(directory: "public") do
+      stub_running
+      run_command("boot", config: :with_error_pages).tap do |output|
+        assert_match /Uploading .*kamal-error-pages.*\/latest to \.kamal\/proxy\/apps-config\/app\/error_pages/, output
+        assert_match "docker tag dhh/app:latest dhh/app:latest", output
+        assert_match /docker run --detach --restart unless-stopped --name app-web-latest --network kamal --hostname 1.1.1.1-[0-9a-f]{12} /, output
+        assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+        assert_match "Running /usr/bin/env find .kamal/proxy/apps-config/app/error_pages -mindepth 1 -maxdepth 1 ! -name latest -exec rm -rf {} + on 1.1.1.1", output
+      end
+    end
+  end
+
   test "start" do
     SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("999") # old version
 
@@ -244,9 +272,11 @@ class CliAppTest < CliTestCase
 
   test "remove" do
     run_command("remove").tap do |output|
-      assert_match /#{Regexp.escape("sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop")}/, output
+      assert_match "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop", output
-      assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output
+      assert_match "docker container prune --force --filter label=service=app", output
-      assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output
+      assert_match "docker image prune --all --force --filter label=service=app", output
+      assert_match "rm -r .kamal/apps/app on 1.1.1.1", output
+      assert_match "rm -r .kamal/proxy/apps-config/app on 1.1.1.1", output
     end
   end
 
@@ -268,8 +298,16 @@ class CliAppTest < CliTestCase
     end
   end
 
+  test "remove_app_directories" do
+    run_command("remove_app_directories").tap do |output|
+      assert_match "rm -r .kamal/apps/app on 1.1.1.1", output
+      assert_match "rm -r .kamal/proxy/apps-config/app on 1.1.1.1", output
+    end
+  end
+
   test "exec" do
     run_command("exec", "ruby -v").tap do |output|
+      assert_match "docker login -u [REDACTED] -p [REDACTED]", output
       assert_match "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:latest ruby -v", output
     end
   end
@@ -312,18 +350,25 @@ class CliAppTest < CliTestCase
   end
 
   test "exec interactive" do
+    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     SSHKit::Backend::Abstract.any_instance.expects(:exec)
       .with("ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:latest ruby -v'")
+
     run_command("exec", "-i", "ruby -v").tap do |output|
+      assert_hook_ran "pre-connect", output
+      assert_match "docker login -u [REDACTED] -p [REDACTED]", output
       assert_match "Get most recent version available as an image...", output
       assert_match "Launching interactive command with version latest via SSH from new container on 1.1.1.1...", output
     end
   end
 
   test "exec interactive with reuse" do
+    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
     SSHKit::Backend::Abstract.any_instance.expects(:exec)
       .with("ssh -t root@1.1.1.1 -p 22 'docker exec -it app-web-999 ruby -v'")
+
     run_command("exec", "-i", "--reuse", "ruby -v").tap do |output|
+      assert_hook_ran "pre-connect", output
       assert_match "Get current version of running container...", output
       assert_match "Running /usr/bin/env sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done on 1.1.1.1", output
       assert_match "Launching interactive command with version 999 via SSH from existing container on 1.1.1.1...", output
@@ -437,6 +482,24 @@ class CliAppTest < CliTestCase
     end
   end
 
+  test "live" do
+    run_command("live").tap do |output|
+      assert_match "docker exec kamal-proxy kamal-proxy resume app-web on 1.1.1.1", output
+    end
+  end
+
+  test "maintenance" do
+    run_command("maintenance").tap do |output|
+      assert_match "docker exec kamal-proxy kamal-proxy stop app-web --drain-timeout=\"30s\" on 1.1.1.1", output
+    end
+  end
+
+  test "maintenance with options" do
+    run_command("maintenance", "--message", "Hello", "--drain_timeout", "10").tap do |output|
+      assert_match "docker exec kamal-proxy kamal-proxy stop app-web --drain-timeout=\"10s\" --message=\"Hello\" on 1.1.1.1", output
+    end
+  end
+
   private
     def run_command(*command, config: :with_accessories, host: "1.1.1.1", allow_execute_error: false)
       stdouted do

test/cli/build_test.rb

@@ -57,6 +57,7 @@ class CliBuildTest < CliTestCase
       stub_setup
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with { |*args| args[0..1] == [ :docker, :login ] }
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
         .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd, "--recurse-submodules")
@@ -103,6 +104,7 @@ class CliBuildTest < CliTestCase
       stub_setup
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute).with { |*args| args[0..1] == [ :docker, :login ] }
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
         .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd, "--recurse-submodules")
@@ -139,6 +141,9 @@ class CliBuildTest < CliTestCase
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
         .with(:docker, "--version", "&&", :docker, :buildx, "version")
 
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| args[0..1] == [ :docker, :login ] }
+
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
         .with(:docker, :buildx, :rm, "kamal-local-docker-container")
 
@@ -321,7 +326,7 @@ class CliBuildTest < CliTestCase
 
   private
     def run_command(*command, fixture: :with_accessories)
-      stdouted { Kamal::Cli::Build.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) }
+      stdouted { stderred { Kamal::Cli::Build.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) } }
     end
 
     def stub_dependency_checks

test/cli/main_test.rb

@@ -21,7 +21,6 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
     # deploy
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -32,7 +31,6 @@ class CliMainTest < CliTestCase
       assert_match /Ensure Docker is installed.../, output
       # deploy
       assert_match /Acquiring the deploy lock/, output
-      assert_match /Log into image registry/, output
       assert_match /Pull app image/, output
       assert_match /Ensure kamal-proxy is running/, output
       assert_match /Detect stale containers/, output
@@ -45,8 +43,7 @@ class CliMainTest < CliTestCase
     with_test_secrets("secrets" => "DB_PASSWORD=secret") do
       invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true }
 
-      Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
+        Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
-      Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
       Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
       Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
       Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
@@ -56,8 +53,7 @@ class CliMainTest < CliTestCase
 
       run_command("deploy", "--verbose").tap do |output|
         assert_hook_ran "pre-connect", output
-        assert_match /Log into image registry/, output
+          assert_match /Build and push app image/, output
-        assert_match /Build and push app image/, output
         assert_hook_ran "pre-deploy", output
         assert_match /Ensure kamal-proxy is running/, output
         assert_match /Detect stale containers/, output
@@ -70,7 +66,6 @@ class CliMainTest < CliTestCase
   test "deploy with skip_push" do
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -79,7 +74,6 @@ class CliMainTest < CliTestCase
 
     run_command("deploy", "--skip_push").tap do |output|
       assert_match /Acquiring the deploy lock/, output
-      assert_match /Log into image registry/, output
       assert_match /Pull app image/, output
       assert_match /Ensure kamal-proxy is running/, output
       assert_match /Detect stale containers/, output
@@ -153,11 +147,11 @@ class CliMainTest < CliTestCase
     end
   end
 
-  test "deploy errors during outside section leave remove lock" do
+  test "deploy errors during outside section leave remote lock" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, :skip_local => false }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Kamal::Cli::Main.any_instance.expects(:invoke)
-      .with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
+      .with("kamal:cli:build:deliver", [], invoke_options)
       .raises(RuntimeError)
 
     assert_not KAMAL.holding_lock?
@@ -170,7 +164,6 @@ class CliMainTest < CliTestCase
   test "deploy with skipped hooks" do
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => true }
 
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -185,7 +178,6 @@ class CliMainTest < CliTestCase
   test "deploy with missing secrets" do
     invoke_options = { "config_file" => "test/fixtures/deploy_with_secrets.yml", "version" => "999", "skip_hooks" => false }
 
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))

test/cli/proxy_test.rb

@@ -4,21 +4,21 @@ class CliProxyTest < CliTestCase
   test "boot" do
     run_command("boot").tap do |output|
       assert_match "docker login", output
-      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
+      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
     end
   end
 
   test "boot old version" do
     Thread.report_on_exception = false
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2")
+      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :awk, "-F:", "'{print $NF}'")
       .returns("v0.0.1")
       .at_least_once
 
     exception = assert_raises do
       run_command("boot").tap do |output|
         assert_match "docker login", output
-        assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
+        assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
       end
     end
 
@@ -30,13 +30,13 @@ class CliProxyTest < CliTestCase
   test "boot correct version" do
     Thread.report_on_exception = false
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2")
+      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :awk, "-F:", "'{print $NF}'")
       .returns(Kamal::Configuration::PROXY_MINIMUM_VERSION)
       .at_least_once
 
     run_command("boot").tap do |output|
       assert_match "docker login", output
-      assert_match "docker container start kamal-proxy || echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
+      assert_match "docker container start kamal-proxy || echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
     end
   ensure
     Thread.report_on_exception = false
@@ -56,12 +56,12 @@ class CliProxyTest < CliTestCase
     run_command("reboot", "-y").tap do |output|
       assert_match "docker container stop kamal-proxy on 1.1.1.1", output
       assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.1", output
-      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy on 1.1.1.1", output
+      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config on 1.1.1.1", output
       assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target=\"abcdefabcdef:80\" --deploy-timeout=\"6s\" --drain-timeout=\"30s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\" on 1.1.1.1", output
 
       assert_match "docker container stop kamal-proxy on 1.1.1.2", output
       assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.2", output
-      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy on 1.1.1.2", output
+      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config on 1.1.1.2", output
       assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target=\"abcdefabcdef:80\" --deploy-timeout=\"6s\" --drain-timeout=\"30s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\" on 1.1.1.2", output
     end
   end
@@ -179,7 +179,7 @@ class CliProxyTest < CliTestCase
     SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("12345678")
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2")
+      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :awk, "-F:", "'{print $NF}'")
       .returns(Kamal::Configuration::PROXY_MINIMUM_VERSION)
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -196,7 +196,7 @@ class CliProxyTest < CliTestCase
       assert_match "/usr/bin/env mkdir -p .kamal", output
       assert_match "docker network create kamal", output
       assert_match "docker login -u [REDACTED] -p [REDACTED]", output
-      assert_match "docker container start kamal-proxy || echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
+      assert_match "docker container start kamal-proxy || echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output
       assert_match "/usr/bin/env mkdir -p .kamal", output
       assert_match %r{docker rename app-web-latest app-web-latest_replaced_.*}, output
       assert_match "/usr/bin/env mkdir -p .kamal/apps/app/env/roles", output
@@ -218,7 +218,7 @@ class CliProxyTest < CliTestCase
     SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("12345678")
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2")
+      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :awk, "-F:", "'{print $NF}'")
       .returns(Kamal::Configuration::PROXY_MINIMUM_VERSION)
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -238,8 +238,9 @@ class CliProxyTest < CliTestCase
     run_command("boot_config", "set").tap do |output|
       %w[ 1.1.1.1 1.1.1.2 ].each do |host|
         assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
-        assert_match "Uploading \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/options on #{host}", output
-      end
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output      end
     end
   end
 
@@ -248,6 +249,8 @@ class CliProxyTest < CliTestCase
       %w[ 1.1.1.1 1.1.1.2 ].each do |host|
         assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
         assert_match "Uploading \"--log-opt max-size=10m\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
       end
     end
   end
@@ -257,6 +260,8 @@ class CliProxyTest < CliTestCase
       %w[ 1.1.1.1 1.1.1.2 ].each do |host|
         assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
         assert_match "Uploading \"--publish 80:80 --publish 443:443 --log-opt max-size=100m\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
       end
     end
   end
@@ -266,6 +271,8 @@ class CliProxyTest < CliTestCase
       %w[ 1.1.1.1 1.1.1.2 ].each do |host|
         assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
         assert_match "Uploading \"--publish 80:80 --publish 443:443\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
       end
     end
   end
@@ -310,19 +317,64 @@ class CliProxyTest < CliTestCase
       %w[ 1.1.1.1 1.1.1.2 ].each do |host|
         assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
         assert_match "Uploading \"--publish 80:80 --publish 443:443 --log-opt max-size=10m --label=foo=bar --add_host=thishost:thathost\" to .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+      end
+    end
+  end
+
+  test "boot_config set registry" do
+    run_command("boot_config", "set", "--registry", "myreg").tap do |output|
+      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
+        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/options on #{host}", output
+        assert_match "Uploading \"myreg/basecamp/kamal-proxy\" to .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+      end
+    end
+  end
+
+  test "boot_config set repository" do
+    run_command("boot_config", "set", "--repository", "myrepo").tap do |output|
+      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
+        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/options on #{host}", output
+        assert_match "Uploading \"myrepo/kamal-proxy\" to .kamal/proxy/image on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image_version on #{host}", output
+      end
+    end
+  end
+
+  test "boot_config set image_version" do
+    run_command("boot_config", "set", "--image_version", "0.9.9").tap do |output|
+      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
+        assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/options on #{host}", output
+        assert_match "Running /usr/bin/env rm .kamal/proxy/image on #{host}", output
+        assert_match "Uploading \"0.9.9\" to .kamal/proxy/image_version on #{host}", output
+      end
+    end
+  end
+
+  test "boot_config set all" do
+    run_command("boot_config", "set", "--docker_options", "label=foo=bar", "--registry", "myreg", "--repository", "myrepo", "--image_version", "0.9.9").tap do |output|
+      %w[ 1.1.1.1 1.1.1.2 ].each do |host|
+        assert_match "Uploading \"--publish 80:80 --publish 443:443 --log-opt max-size=10m --label=foo=bar\" to .kamal/proxy/options on #{host}", output
+        assert_match "Uploading \"myreg/myrepo/kamal-proxy\" to .kamal/proxy/image on #{host}", output
+        assert_match "Uploading \"0.9.9\" to .kamal/proxy/image_version on #{host}", output
       end
     end
   end
 
   test "boot_config get" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:cat, ".kamal/proxy/options", "2>", "/dev/null", "||", :echo, "\"--publish 80:80 --publish 443:443 --log-opt max-size=10m\"")
+      .with(:echo, "$(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"v0.8.7\")")
-      .returns("--publish 80:80 --publish 8443:443 --label=foo=bar")
+      .returns("--publish 80:80 --publish 8443:443 --label=foo=bar basecamp/kamal-proxy:v1.0.0")
       .twice
 
     run_command("boot_config", "get").tap do |output|
-      assert_match "Host 1.1.1.1: --publish 80:80 --publish 8443:443 --label=foo=bar", output
+      assert_match "Host 1.1.1.1: --publish 80:80 --publish 8443:443 --label=foo=bar basecamp/kamal-proxy:v1.0.0", output
-      assert_match "Host 1.1.1.2: --publish 80:80 --publish 8443:443 --label=foo=bar", output
+      assert_match "Host 1.1.1.2: --publish 80:80 --publish 8443:443 --label=foo=bar basecamp/kamal-proxy:v1.0.0", output
     end
   end
 

test/commands/app_test.rb

@@ -497,6 +497,30 @@ class CommandsAppTest < ActiveSupport::TestCase
     ], new_command(asset_path: "/public/assets").clean_up_assets
   end
 
+  test "live" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy resume app-web",
+      new_command.live.join(" ")
+  end
+
+  test "maintenance" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy stop app-web",
+      new_command.maintenance.join(" ")
+  end
+
+  test "maintenance with options" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy stop app-web --drain-timeout=\"10s\" --message=\"Hi\"",
+      new_command.maintenance(drain_timeout: 10, message: "Hi").join(" ")
+  end
+
+  test "remove_proxy_app_directory" do
+    assert_equal \
+      "rm -r .kamal/proxy/apps-config/app",
+      new_command.remove_proxy_app_directory.join(" ")
+  end
+
   private
     def new_command(role: "web", host: "1.1.1.1", **additional_config)
       config = Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999")

test/commands/proxy_test.rb

@@ -15,7 +15,7 @@ class CommandsProxyTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy",
+      "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config",
       new_command.run.join(" ")
   end
 
@@ -23,7 +23,7 @@ class CommandsProxyTest < ActiveSupport::TestCase
     @config.delete(:proxy)
 
     assert_equal \
-      "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy",
+      "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config",
       new_command.run.join(" ")
   end
 
@@ -101,7 +101,7 @@ class CommandsProxyTest < ActiveSupport::TestCase
 
   test "version" do
     assert_equal \
-      "docker inspect kamal-proxy --format '{{.Config.Image}}' | cut -d: -f2",
+      "docker inspect kamal-proxy --format '{{.Config.Image}}' | awk -F: '{print $NF}'",
       new_command.version.join(" ")
   end
 
@@ -111,10 +111,22 @@ class CommandsProxyTest < ActiveSupport::TestCase
       new_command.ensure_proxy_directory.join(" ")
   end
 
-  test "get_boot_options" do
+  test "read_boot_options" do
     assert_equal \
       "cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\"",
-      new_command.get_boot_options.join(" ")
+      new_command.read_boot_options.join(" ")
+  end
+
+  test "read_image" do
+    assert_equal \
+      "cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"",
+      new_command.read_image.join(" ")
+  end
+
+  test "read_image_version" do
+    assert_equal \
+      "cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\"",
+      new_command.read_image_version.join(" ")
   end
 
   test "reset_boot_options" do
@@ -123,6 +135,18 @@ class CommandsProxyTest < ActiveSupport::TestCase
       new_command.reset_boot_options.join(" ")
   end
 
+  test "reset_image" do
+    assert_equal \
+      "rm .kamal/proxy/image",
+      new_command.reset_image.join(" ")
+  end
+
+  test "reset_image_version" do
+    assert_equal \
+      "rm .kamal/proxy/image_version",
+      new_command.reset_image_version.join(" ")
+  end
+
   private
     def new_command
       Kamal::Commands::Proxy.new(Kamal::Configuration.new(@config, version: "123"))

test/configuration/accessory_test.rb

@@ -187,4 +187,12 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
     assert @config.accessory(:monitoring).running_proxy?
     assert_equal [ "monitoring.example.com" ], @config.accessory(:monitoring).proxy.hosts
   end
+
+  test "can't set restart in options" do
+    @deploy[:accessories]["mysql"]["options"] = { "restart" => "always" }
+
+    assert_raises Kamal::ConfigurationError, "servers/workers: Cannot set restart policy in docker options, unless-stopped is required" do
+      Kamal::Configuration.new(@deploy)
+    end
+  end
 end

test/configuration/proxy_test.rb

@@ -38,6 +38,13 @@ class ConfigurationProxyTest < ActiveSupport::TestCase
     assert_not config.proxy.ssl?
   end
 
+  test "false not allowed" do
+    @deploy[:proxy] = false
+    assert_raises(Kamal::ConfigurationError, "proxy: should be a hash") do
+      config.proxy
+    end
+  end
+
   private
     def config
       Kamal::Configuration.new(@deploy)

test/configuration/role_test.rb

@@ -258,6 +258,14 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     assert_equal "18s", config_with_roles.role(:workers).proxy.deploy_options[:"target-timeout"]
   end
 
+  test "can't set restart in options" do
+    @deploy_with_roles[:servers]["workers"]["options"] = { "restart" => "always" }
+
+    assert_raises Kamal::ConfigurationError, "servers/workers: Cannot set restart policy in docker options, unless-stopped is required" do
+      Kamal::Configuration.new(@deploy_with_roles)
+    end
+  end
+
   private
     def config
       Kamal::Configuration.new(@deploy)

test/configuration_test.rb

@@ -30,7 +30,7 @@ class ConfigurationTest < ActiveSupport::TestCase
   %i[ service image registry ].each do |key|
     test "#{key} config required" do
       assert_raise(Kamal::ConfigurationError) do
-        Kamal::Configuration.new @deploy.tap { _1.delete key }
+        Kamal::Configuration.new @deploy.tap { |config| config.delete key }
       end
     end
   end
@@ -38,21 +38,36 @@ class ConfigurationTest < ActiveSupport::TestCase
   %w[ username password ].each do |key|
     test "registry #{key} required" do
       assert_raise(Kamal::ConfigurationError) do
-        Kamal::Configuration.new @deploy.tap { _1[:registry].delete key }
+        Kamal::Configuration.new @deploy.tap { |config| config[:registry].delete key }
       end
     end
   end
 
   test "service name valid" do
     assert_nothing_raised do
-      Kamal::Configuration.new(@deploy.tap { _1[:service] = "hey-app1_primary" })
+      Kamal::Configuration.new(@deploy.tap { |config| config[:service] = "hey-app1_primary" })
-      Kamal::Configuration.new(@deploy.tap { _1[:service] = "MyApp" })
+      Kamal::Configuration.new(@deploy.tap { |config| config[:service] = "MyApp" })
     end
   end
 
   test "service name invalid" do
     assert_raise(Kamal::ConfigurationError) do
-      Kamal::Configuration.new @deploy.tap { _1[:service] = "app.com" }
+      Kamal::Configuration.new @deploy.tap { |config| config[:service] = "app.com" }
+    end
+  end
+
+  test "servers required" do
+    assert_raise(Kamal::ConfigurationError) do
+      Kamal::Configuration.new @deploy.tap { |config| config.delete(:servers) }
+    end
+  end
+
+  test "servers not required with accessories" do
+    assert_nothing_raised do
+      @deploy.delete(:servers)
+      @deploy[:accessories] = { "foo" => { "image" => "foo/bar", "host" => "1.1.1.1" } }
+
+      Kamal::Configuration.new(@deploy)
     end
   end
 
@@ -250,7 +265,7 @@ class ConfigurationTest < ActiveSupport::TestCase
   test "destination required" do
     dest_config_file = Pathname.new(File.expand_path("fixtures/deploy_for_required_dest.yml", __dir__))
 
-    assert_raises(Kamal::ConfigurationError) do
+    assert_raises(ArgumentError, "You must specify a destination") do
       config = Kamal::Configuration.create_from config_file: dest_config_file
     end
 
@@ -395,4 +410,13 @@ class ConfigurationTest < ActiveSupport::TestCase
 
     assert_equal "Different roles can't share the same host for SSL: foo.example.com", exception.message
   end
+
+  test "proxy directories" do
+    assert_equal ".kamal/proxy/apps-config", @config.proxy_apps_directory
+    assert_equal "/home/kamal-proxy/.apps-config", @config.proxy_apps_container_directory
+    assert_equal ".kamal/proxy/apps-config/app", @config.proxy_app_directory
+    assert_equal "/home/kamal-proxy/.apps-config/app", @config.proxy_app_container_directory
+    assert_equal ".kamal/proxy/apps-config/app/error_pages", @config.proxy_error_pages_directory
+    assert_equal "/home/kamal-proxy/.apps-config/app/error_pages", @config.proxy_error_pages_container_directory
+  end
 end

test/fixtures/deploy_with_error_pages.yml

@@ -0,0 +1,11 @@
+service: app
+image: dhh/app
+servers:
+  - "1.1.1.1"
+  - "1.1.1.2"
+registry:
+  username: user
+  password: pw
+builder:
+  arch: amd64
+error_pages_path: public

test/integration/accessory_test.rb

@@ -17,10 +17,39 @@ class AccessoryTest < IntegrationTest
     logs = kamal :accessory, :logs, :busybox, capture: true
     assert_match /Starting busybox.../, logs
 
+    boot = kamal :accessory, :boot, :busybox, capture: true
+    assert_match /Skipping booting `busybox` on vm1, vm2, a container already exists/, boot
+
     kamal :accessory, :remove, :busybox, "-y"
     assert_accessory_not_running :busybox
   end
 
+  test "proxied: boot, stop, start, restart, logs, remove" do
+    @app = "app_with_proxied_accessory"
+
+    kamal :proxy, :boot
+
+    kamal :accessory, :boot, :netcat
+    assert_accessory_running :netcat
+    assert_netcat_is_up
+
+    kamal :accessory, :stop, :netcat
+    assert_accessory_not_running :netcat
+    assert_netcat_not_found
+
+    kamal :accessory, :start, :netcat
+    assert_accessory_running :netcat
+    assert_netcat_is_up
+
+    kamal :accessory, :restart, :netcat
+    assert_accessory_running :netcat
+    assert_netcat_is_up
+
+    kamal :accessory, :remove, :netcat, "-y"
+    assert_accessory_not_running :netcat
+    assert_netcat_not_found
+  end
+
   private
     def assert_accessory_running(name)
       assert_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
@@ -33,4 +62,25 @@ class AccessoryTest < IntegrationTest
     def accessory_details(name)
       kamal :accessory, :details, name, capture: true
     end
+
+    def assert_netcat_is_up
+      response = netcat_response
+      debug_response_code(response, "200")
+      assert_equal "200", response.code
+    end
+
+    def assert_netcat_not_found
+      response = netcat_response
+      debug_response_code(response, "404")
+      assert_equal "404", response.code
+    end
+
+    def netcat_response
+      uri = URI.parse("http://127.0.0.1:12345/up")
+      http = Net::HTTP.new(uri.host, uri.port)
+      request = Net::HTTP::Get.new(uri)
+      request["Host"] = "netcat"
+
+      http.request(request)
+    end
 end

test/integration/app_test.rb

@@ -48,9 +48,36 @@ class AppTest < IntegrationTest
     assert_match "App Host: vm1", exec_output
     assert_match /1 root      0:\d\d nginx/, exec_output
 
+    kamal :app, :maintenance
+    assert_app_in_maintenance
+
+    kamal :app, :live
+    assert_app_is_up
+
     kamal :app, :remove
 
     assert_app_not_found
     assert_app_directory_removed
   end
+
+  test "custom error pages" do
+    @app = "app_with_roles"
+
+    kamal :deploy
+    assert_app_is_up
+
+    kamal :app, :maintenance
+    assert_app_in_maintenance message: "Custom Maintenance Page"
+
+    kamal :app, :live
+    kamal :app, :maintenance, "--message", "\"Testing Maintence Mode\""
+    assert_app_in_maintenance message: "Custom Maintenance Page: Testing Maintence Mode"
+
+    second_version = update_app_rev
+
+    kamal :redeploy
+
+    kamal :app, :maintenance
+    assert_app_in_maintenance message: "Custom Maintenance Page"
+  end
 end

test/integration/docker/deployer/app/.kamal/hooks/pre-deploy

@@ -1,3 +1,6 @@
 #!/bin/sh
+set -e
+
+kamal proxy boot_config set --registry registry:4443
 echo "Deployed!"
 mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-deploy

test/integration/docker/deployer/app/config/deploy.yml

@@ -41,3 +41,8 @@ accessories:
     cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done'
     roles:
       - web
+  busybox2:
+    service: custom-busybox
+    image: registry:4443/busybox:1.36.0
+    cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done'
+    host: vm3

test/integration/docker/deployer/app_with_proxied_accessory/config/deploy.yml

@@ -1,7 +1,5 @@
 service: app_with_proxied_accessory
 image: app_with_proxied_accessory
-servers:
-  - vm1
 env:
   clear:
     CLEAR_TOKEN: 4321
@@ -24,15 +22,13 @@ accessories:
     service: custom-busybox
     image: registry:4443/busybox:1.36.0
     cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done'
-    roles:
+    host: vm1
-      - web
   netcat:
     service: netcat
     image: registry:4443/busybox:1.36.0
     cmd: >
       sh -c 'echo "Starting netcat..."; while true; do echo -e "HTTP/1.1 200 OK\r\nContent-Length: 11\r\n\r\nHello Ruby" | nc -l -p 80; done'
-    roles:
+    host: vm1
-      - web
     port: 12345:80
     proxy:
       host: netcat
@@ -41,4 +37,4 @@ accessories:
         interval: 1
         timeout: 1
         path: "/"
-
+drain_timeout: 2

test/integration/docker/deployer/app_with_roles/.kamal/hooks/pre-deploy

@@ -1,3 +1,6 @@
 #!/bin/sh
+set -e
+
+kamal proxy boot_config set --registry registry:4443
 echo "Deployed!"
 mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-deploy

test/integration/docker/deployer/app_with_roles/config/deploy.yml

@@ -37,6 +37,7 @@ proxy:
       - X-Request-Start
 
 asset_path: /usr/share/nginx/html/versions
+error_pages_path: error_pages
 
 registry:
   server: registry:4443

test/integration/docker/deployer/app_with_roles/error_pages/503.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>503 Service Interrupted</title>
+  </head>
+  <body>
+    <p>Custom Maintenance Page: {{ .Message }}</p>
+  </body>
+</html>

test/integration/docker/deployer/app_with_traefik/.kamal/hooks/pre-deploy

@@ -1,4 +1,7 @@
-kamal proxy boot_config set --publish false \
+set -e
+
+kamal proxy boot_config set --registry registry:4443 \
+                            --publish false \
                             --docker_options label=traefik.http.services.kamal_proxy.loadbalancer.server.scheme=http \
                                              label=traefik.http.routers.kamal_proxy.rule=PathPrefix\(\`/\`\) \
                                              sysctl=net.ipv4.ip_local_port_range=\"10000\ 60999\"

test/integration/docker/deployer/setup.sh

@@ -20,6 +20,7 @@ push_image_to_registry_4443() {
 install_kamal
 push_image_to_registry_4443 nginx 1-alpine-slim
 push_image_to_registry_4443 busybox 1.36.0
+push_image_to_registry_4443 basecamp/kamal-proxy v0.8.7
 
 # .ssh is on a shared volume that persists between runs. Clean it up as the
 # churn of temporary vm IPs can eventually create conflicts.

test/integration/docker/registry/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry
+FROM registry:3
 
 COPY boot.sh .
 

test/integration/docker/registry/boot.sh

@@ -2,4 +2,4 @@
 
 while [ ! -f /certs/domain.crt ]; do sleep 1; done
 
-exec /entrypoint.sh /etc/docker/registry/config.yml
+exec /entrypoint.sh /etc/distribution/config.yml

test/integration/integration_test.rb

@@ -11,7 +11,7 @@ class IntegrationTest < ActiveSupport::TestCase
   end
 
   teardown do
-    unless passed?
+    if !passed? && ENV["DEBUG_CONTAINER_LOGS"]
       [ :deployer, :vm1, :vm2, :shared, :load_balancer, :registry ].each do |container|
         puts
         puts "Logs for #{container}:"
@@ -25,8 +25,8 @@ class IntegrationTest < ActiveSupport::TestCase
     def docker_compose(*commands, capture: false, raise_on_error: true)
       command = "TEST_ID=#{ENV["TEST_ID"]} docker compose #{commands.join(" ")}"
       succeeded = false
-      if capture
+      if capture || !ENV["DEBUG"]
-        result = stdouted { succeeded = system("cd test/integration && #{command}") }
+        result = stdouted { stderred { succeeded = system("cd test/integration && #{command}") } }
       else
         succeeded = system("cd test/integration && #{command}")
       end
@@ -45,15 +45,22 @@ class IntegrationTest < ActiveSupport::TestCase
     end
 
     def assert_app_is_down
-      response = app_response
+      assert_app_error_code("502")
-      debug_response_code(response, "502")
+    end
-      assert_equal "502", response.code
+
+    def assert_app_in_maintenance(message: nil)
+      assert_app_error_code("503", message: message)
     end
 
     def assert_app_not_found
+      assert_app_error_code("404")
+    end
+
+    def assert_app_error_code(code, message: nil)
       response = app_response
-      debug_response_code(response, "404")
+      debug_response_code(response, code)
-      assert_equal "404", response.code
+      assert_equal code, response.code
+      assert_match message, response.body.strip if message
     end
 
     def assert_app_is_up(version: nil, app: @app)

test/integration/main_test.rb

@@ -60,7 +60,7 @@ class MainTest < IntegrationTest
     version = latest_app_version
 
     assert_equal [ "web" ], config[:roles]
-    assert_equal [ "vm1", "vm2" ], config[:hosts]
+    assert_equal [ "vm1", "vm2", "vm3" ], config[:hosts]
     assert_equal "vm1", config[:primary_host]
     assert_equal version, config[:version]
     assert_equal "registry:4443/app", config[:repository]
@@ -88,8 +88,6 @@ class MainTest < IntegrationTest
   end
 
   test "setup and remove" do
-    @app = "app_with_roles"
-
     kamal :proxy, :boot_config, "set",
       "--publish=false",
       "--docker-options=label=traefik.http.services.kamal_proxy.loadbalancer.server.scheme=http",
@@ -172,21 +170,25 @@ class MainTest < IntegrationTest
       assert_equal "200", Net::HTTP.get_response(URI.parse("http://#{app_host}:12345/versions/.hidden")).code
     end
 
-    def vm1_image_ids
+    def image_ids(vm:)
-      docker_compose("exec vm1 docker image ls -q", capture: true).strip.split("\n")
+      docker_compose("exec #{vm} docker image ls -q", capture: true).strip.split("\n")
     end
 
-    def vm1_container_ids
+    def container_ids(vm:)
-      docker_compose("exec vm1 docker ps -a -q", capture: true).strip.split("\n")
+      docker_compose("exec #{vm} docker ps -a -q", capture: true).strip.split("\n")
     end
 
     def assert_no_images_or_containers
-      assert vm1_image_ids.empty?
+      [ :vm1, :vm2, :vm3 ].each do |vm|
-      assert vm1_container_ids.empty?
+        assert image_ids(vm: vm).empty?
+        assert container_ids(vm: vm).empty?
+      end
     end
 
     def assert_images_and_containers
-      assert vm1_image_ids.any?
+      [ :vm1, :vm2, :vm3 ].each do |vm|
-      assert vm1_container_ids.any?
+        assert image_ids(vm: vm).any?
+        assert container_ids(vm: vm).any?
+      end
     end
 end

test/integration/proxied_accessory_test.rb

@@ -1,63 +0,0 @@
-require_relative "integration_test"
-
-class ProxiedAccessoryTest < IntegrationTest
-  test "boot, stop, start, restart, logs, remove" do
-    @app = "app_with_proxied_accessory"
-
-    kamal :deploy
-
-    kamal :accessory, :boot, :netcat
-    assert_accessory_running :netcat
-    assert_netcat_is_up
-
-    kamal :accessory, :stop, :netcat
-    assert_accessory_not_running :netcat
-    assert_netcat_not_found
-
-    kamal :accessory, :start, :netcat
-    assert_accessory_running :netcat
-    assert_netcat_is_up
-
-    kamal :accessory, :restart, :netcat
-    assert_accessory_running :netcat
-    assert_netcat_is_up
-
-    kamal :accessory, :remove, :netcat, "-y"
-    assert_accessory_not_running :netcat
-    assert_netcat_not_found
-  end
-
-  private
-    def assert_accessory_running(name)
-      assert_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
-    end
-
-    def assert_accessory_not_running(name)
-      assert_no_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
-    end
-
-    def accessory_details(name)
-      kamal :accessory, :details, name, capture: true
-    end
-
-    def assert_netcat_is_up
-      response = netcat_response
-      debug_response_code(response, "200")
-      assert_equal "200", response.code
-    end
-
-    def assert_netcat_not_found
-      response = netcat_response
-      debug_response_code(response, "404")
-      assert_equal "404", response.code
-    end
-
-    def netcat_response
-      uri = URI.parse("http://127.0.0.1:12345/up")
-      http = Net::HTTP.new(uri.host, uri.port)
-      request = Net::HTTP::Get.new(uri)
-      request["Host"] = "netcat"
-
-      http.request(request)
-    end
-end

test/integration/proxy_test.rb

@@ -6,6 +6,8 @@ class ProxyTest < IntegrationTest
   end
 
   test "boot, reboot, stop, start, restart, logs, remove" do
+    kamal :proxy, :boot_config, :set, "--registry", "registry:4443"
+
     kamal :proxy, :boot
     assert_proxy_running
 
@@ -46,7 +48,7 @@ class ProxyTest < IntegrationTest
     logs = kamal :proxy, :logs, capture: true
     assert_match /No previous state to restore/, logs
 
-    kamal :proxy, :boot_config, :set, "--docker-options='sysctl net.ipv4.ip_local_port_range=\"10000 60999\"'"
+    kamal :proxy, :boot_config, :set, "--registry", "registry:4443", "--docker-options='sysctl net.ipv4.ip_local_port_range=\"10000 60999\"'"
     assert_docker_options_in_file
 
     kamal :proxy, :reboot, "-y"

test/secrets/aws_secrets_manager_adapter_test.rb

@@ -4,7 +4,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
   test "fails when errors are present" do
     stub_ticks.with("aws --version 2> /dev/null")
     stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list unknown1 unknown2 --profile default")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list unknown1 unknown2 --profile default --output json")
       .returns(<<~JSON)
         {
           "SecretValues": [],
@@ -33,7 +33,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
   test "fetch" do
     stub_ticks.with("aws --version 2> /dev/null")
     stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 secret2/KEY3 --profile default")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 secret2/KEY3 --profile default --output json")
       .returns(<<~JSON)
         {
           "SecretValues": [
@@ -76,7 +76,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
   test "fetch with string value" do
     stub_ticks.with("aws --version 2> /dev/null")
     stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret secret2/KEY1 --profile default")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret secret2/KEY1 --profile default --output json")
       .returns(<<~JSON)
         {
           "SecretValues": [
@@ -118,7 +118,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
   test "fetch with secret names" do
     stub_ticks.with("aws --version 2> /dev/null")
     stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 --profile default")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 --profile default --output json")
       .returns(<<~JSON)
         {
           "SecretValues": [
@@ -159,7 +159,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase
   test "fetch without account option omits --profile" do
     stub_ticks.with("aws --version 2> /dev/null")
     stub_ticks
-      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2")
+      .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 --output json")
       .returns(<<~JSON)
         {
           "SecretValues": [

test/test_helper.rb

@@ -58,9 +58,7 @@ class ActiveSupport::TestCase
     def setup_test_secrets(**files)
       @original_pwd = Dir.pwd
       @secrets_tmpdir = Dir.mktmpdir
-      fixtures_dup = File.join(@secrets_tmpdir, "test")
+      copy_fixtures(@secrets_tmpdir)
-      FileUtils.mkdir_p(fixtures_dup)
-      FileUtils.cp_r("test/fixtures/", fixtures_dup)
 
       Dir.chdir(@secrets_tmpdir)
       FileUtils.mkdir_p(".kamal")
@@ -75,6 +73,30 @@ class ActiveSupport::TestCase
       Dir.chdir(@original_pwd)
       FileUtils.rm_rf(@secrets_tmpdir)
     end
+
+    def with_error_pages(directory:)
+      error_pages_tmpdir = Dir.mktmpdir
+
+      Dir.mktmpdir do |tmpdir|
+        copy_fixtures(tmpdir)
+
+        Dir.chdir(tmpdir) do
+          FileUtils.mkdir_p(directory)
+          Dir.chdir(directory) do
+            File.write("404.html", "404 page")
+            File.write("503.html", "503 page")
+          end
+
+          yield
+        end
+      end
+    end
+
+    def copy_fixtures(to_dir)
+      new_test_dir = File.join(to_dir, "test")
+      FileUtils.mkdir_p(new_test_dir)
+      FileUtils.cp_r("test/fixtures/", new_test_dir)
+    end
 end
 
 class SecretAdapterTestCase < ActiveSupport::TestCase