Maintenance mode

Adds support for maintenance mode to Kamal.

There are two new commands:
- `kamal app maintenance` - puts the app in maintenance mode
- `kamal app live` - puts the app back in live mode

In maintenance mode, the kamal proxy will respond to requests with a
503 status code. It will use an error page built into kamal proxy.

You can use your own error page by setting `error_pages_path` in the
configuration. This will copy any 4xx.html or 5xx.html files from that
page to a volume mounted into the proxy container.

lib/kamal/cli/app.rb

@@ -8,8 +8,10 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
         # Assets are prepared in a separate step to ensure they are on all hosts before booting
         on(KAMAL.hosts) do
+          Kamal::Cli::App::ErrorPages.new(host, self).run
+
           KAMAL.roles_on(host).each do |role|
-            Kamal::Cli::App::PrepareAssets.new(host, role, self).run
+            Kamal::Cli::App::Assets.new(host, role, self).run
           end
         end
 
@@ -249,7 +251,37 @@ class Kamal::Cli::App < Kamal::Cli::Base
       stop
       remove_containers
       remove_images
-      remove_app_directory
+      remove_app_directories
+    end
+  end
+
+  desc "live", "Set the app to live mode"
+  def live
+    with_lock do
+      on(KAMAL.proxy_hosts) do |host|
+        roles = KAMAL.roles_on(host)
+
+        roles.each do |role|
+          execute *KAMAL.app(role: role, host: host).live if role.running_proxy?
+        end
+      end
+    end
+  end
+
+  desc "maintenance", "Set the app to maintenance mode"
+  option :drain_timeout, type: :numeric, desc: "How long to allow in-flight requests to complete (defaults to drain_timeout from config)"
+  option :message, type: :string, desc: "Message to display to clients while stopped"
+  def maintenance
+    maintenance_options = { drain_timeout: options[:drain_timeout] || KAMAL.config.drain_timeout, message: options[:message] }
+
+    with_lock do
+      on(KAMAL.proxy_hosts) do |host|
+        roles = KAMAL.roles_on(host)
+
+        roles.each do |role|
+          execute *KAMAL.app(role: role, host: host).maintenance(**maintenance_options) if role.running_proxy?
+        end
+      end
     end
   end
 
@@ -291,16 +323,19 @@ class Kamal::Cli::App < Kamal::Cli::Base
     end
   end
 
-  desc "remove_app_directory", "Remove the service directory from servers", hide: true
-  def remove_app_directory
+  desc "remove_app_directories", "Remove the app directories from servers", hide: true
+  def remove_app_directories
     with_lock do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
-          execute *KAMAL.auditor.record("Removed #{KAMAL.config.app_directory} on all servers", role: role), verbosity: :debug
+          execute *KAMAL.auditor.record("Removed #{KAMAL.config.app_directory}", role: role), verbosity: :debug
           execute *KAMAL.server.remove_app_directory, raise_on_non_zero_exit: false
         end
+
+        execute *KAMAL.auditor.record("Removed #{KAMAL.config.app_directory}"), verbosity: :debug
+        execute *KAMAL.app.remove_proxy_app_directory, raise_on_non_zero_exit: false
       end
     end
   end

lib/kamal/cli/app/assets.rb

@@ -1,4 +1,4 @@
-class Kamal::Cli::App::PrepareAssets
+class Kamal::Cli::App::Assets
   attr_reader :host, :role, :sshkit
   delegate :execute, :capture_with_info, :info, to: :sshkit
   delegate :assets?, to: :role

lib/kamal/cli/app/boot.rb

@@ -70,6 +70,7 @@ class Kamal::Cli::App::Boot
     def stop_old_version(version)
       execute *app.stop(version: version), raise_on_non_zero_exit: false
       execute *app.clean_up_assets if assets?
+      execute *app.clean_up_error_pages if KAMAL.config.error_pages_path
     end
 
     def release_barrier

lib/kamal/cli/app/error_pages.rb

@@ -0,0 +1,33 @@
+class Kamal::Cli::App::ErrorPages
+  ERROR_PAGES_GLOB = "{4??.html,5??.html}"
+
+  attr_reader :host, :sshkit
+  delegate :upload!, :execute, to: :sshkit
+
+  def initialize(host, sshkit)
+    @host = host
+    @sshkit = sshkit
+  end
+
+  def run
+    if KAMAL.config.error_pages_path
+      with_error_pages_tmpdir do |local_error_pages_dir|
+        execute *KAMAL.app.create_error_pages_directory
+        upload! local_error_pages_dir, KAMAL.config.proxy_error_pages_directory, mode: "0700", recursive: true
+      end
+    end
+  end
+
+  private
+    def with_error_pages_tmpdir
+      Dir.mktmpdir("kamal-error-pages") do |tmpdir|
+        error_pages_dir = File.join(tmpdir, KAMAL.config.version)
+        FileUtils.mkdir(error_pages_dir)
+
+        if (files = Dir[File.join(KAMAL.config.error_pages_path, ERROR_PAGES_GLOB)]).any?
+          FileUtils.cp(files, error_pages_dir)
+          yield error_pages_dir
+        end
+      end
+    end
+end

lib/kamal/commands/app.rb

@@ -1,5 +1,5 @@
 class Kamal::Commands::App < Kamal::Commands::Base
-  include Assets, Containers, Execution, Images, Logging, Proxy
+  include Assets, Containers, ErrorPages, Execution, Images, Logging, Proxy
 
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 

lib/kamal/commands/app/error_pages.rb

@@ -0,0 +1,9 @@
+module Kamal::Commands::App::ErrorPages
+  def create_error_pages_directory
+    make_directory(config.proxy_error_pages_directory)
+  end
+
+  def clean_up_error_pages
+    [ :find, config.proxy_error_pages_directory, "-mindepth", "1", "-maxdepth", "1", "!", "-name", KAMAL.config.version, "-exec", "rm", "-rf", "{} +" ]
+  end
+end

lib/kamal/commands/app/proxy.rb

@@ -9,6 +9,18 @@ module Kamal::Commands::App::Proxy
     proxy_exec :remove, role.container_prefix
   end
 
+  def live
+    proxy_exec :resume, role.container_prefix
+  end
+
+  def maintenance(**options)
+    proxy_exec :stop, role.container_prefix, *role.proxy.stop_command_args(**options)
+  end
+
+  def remove_proxy_app_directory
+    remove_directory config.proxy_app_directory
+  end
+
   private
     def proxy_exec(*command)
       docker :exec, proxy_container_name, "kamal-proxy", *command

lib/kamal/commands/proxy.rb

@@ -110,6 +110,6 @@ class Kamal::Commands::Proxy < Kamal::Commands::Base
         "--detach",
         "--restart", "unless-stopped",
         "--volume", "kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy",
-        *config.proxy_app_config_volume.docker_args
+        *config.proxy_apps_volume.docker_args
     end
 end

lib/kamal/configuration.rb

@@ -105,6 +105,10 @@ class Kamal::Configuration
     raw_config.minimum_version
   end
 
+  def service_and_destination
+    [ service, destination ].compact.join("-")
+  end
+
   def roles
     servers.roles
   end
@@ -210,7 +214,7 @@ class Kamal::Configuration
   end
 
   def app_directory
-    File.join apps_directory, [ service, destination ].compact.join("-")
+    File.join apps_directory, service_and_destination
   end
 
   def env_directory
@@ -229,6 +233,10 @@ class Kamal::Configuration
     raw_config.asset_path
   end
 
+  def error_pages_path
+    raw_config.error_pages_path
+  end
+
   def env_tags
     @env_tags ||= if (tags = raw_config.env["tags"])
       tags.collect { |name, config| Env::Tag.new(name, config: config, secrets: secrets) }
@@ -300,14 +308,34 @@ class Kamal::Configuration
     File.join proxy_directory, "image_version"
   end
 
-  def proxy_app_config_directory
-    File.join proxy_directory, "app-config"
+  def proxy_apps_directory
+    File.join proxy_directory, "apps-config"
   end
 
-  def proxy_app_config_volume
+  def proxy_apps_container_directory
+    "/home/kamal-proxy/.apps-config"
+  end
+
+  def proxy_apps_volume
     Volume.new \
-      host_path: proxy_app_config_directory,
-      container_path: "/home/kamal-proxy/.app-config"
+      host_path: proxy_apps_directory,
+      container_path: proxy_apps_container_directory
+  end
+
+  def proxy_app_directory
+    File.join proxy_apps_directory, service_and_destination
+  end
+
+  def proxy_app_container_directory
+    File.join proxy_apps_container_directory, service_and_destination
+  end
+
+  def proxy_error_pages_directory
+    File.join proxy_app_directory, "error_pages"
+  end
+
+  def proxy_error_pages_container_directory
+    File.join proxy_app_container_directory, "error_pages"
   end
 
   def to_h

lib/kamal/configuration/docs/configuration.yml

@@ -82,6 +82,12 @@ asset_path: /path/to/assets
 # See https://kamal-deploy.org/docs/hooks for more information:
 hooks_path: /user_home/kamal/hooks
 
+# Error pages
+#
+# A directory relative to the app root to find error pages for the proxy to serve.
+# Any files in the format 4xx.html or 5xx.html will be copied to the hosts.
+error_pages_path: public
+
 # Require destinations
 #
 # Whether deployments require a destination to be specified, defaults to `false`:

lib/kamal/configuration/proxy.rb

@@ -44,7 +44,8 @@ class Kamal::Configuration::Proxy
       "forward-headers": proxy_config.dig("forward_headers"),
       "tls-redirect": proxy_config.dig("ssl_redirect"),
       "log-request-header": proxy_config.dig("logging", "request_headers") || DEFAULT_LOG_REQUEST_HEADERS,
-      "log-response-header": proxy_config.dig("logging", "response_headers")
+      "log-response-header": proxy_config.dig("logging", "response_headers"),
+      "error-pages": error_pages
     }.compact
   end
 
@@ -52,6 +53,17 @@ class Kamal::Configuration::Proxy
     optionize ({ target: "#{target}:#{app_port}" }).merge(deploy_options), with: "="
   end
 
+  def stop_options(drain_timeout: nil, message: nil)
+    {
+      "drain-timeout": seconds_duration(drain_timeout),
+      message: message
+    }.compact
+  end
+
+  def stop_command_args(**options)
+    optionize stop_options(**options), with: "="
+  end
+
   def merge(other)
     self.class.new config: config, proxy_config: proxy_config.deep_merge(other.proxy_config)
   end
@@ -60,4 +72,8 @@ class Kamal::Configuration::Proxy
     def seconds_duration(value)
       value ? "#{value}s" : nil
     end
+
+    def error_pages
+      File.join config.proxy_error_pages_container_directory, config.version if config.error_pages_path
+    end
 end