Maintenance mode

Adds support for maintenance mode to Kamal.

There are two new commands:
- `kamal app maintenance` - puts the app in maintenance mode
- `kamal app live` - puts the app back in live mode

In maintenance mode, the kamal proxy will respond to requests with a
503 status code. It will use an error page built into kamal proxy.

You can use your own error page by setting `error_pages_path` in the
configuration. This will copy any 4xx.html or 5xx.html files from that
page to a volume mounted into the proxy container.

test/cli/app_test.rb

@@ -192,6 +192,19 @@ class CliAppTest < CliTestCase
     Thread.report_on_exception = true
   end
 
+  test "boot with error pages" do
+    with_error_pages(directory: "public") do
+      stub_running
+      run_command("boot", config: :with_error_pages).tap do |output|
+        assert_match /Uploading .*kamal-error-pages.*\/latest to \.kamal\/proxy\/apps-config\/app\/error_pages/, output
+        assert_match "docker tag dhh/app:latest dhh/app:latest", output
+        assert_match /docker run --detach --restart unless-stopped --name app-web-latest --network kamal --hostname 1.1.1.1-[0-9a-f]{12} /, output
+        assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+        assert_match "Running /usr/bin/env find .kamal/proxy/apps-config/app/error_pages -mindepth 1 -maxdepth 1 ! -name latest -exec rm -rf {} + on 1.1.1.1", output
+      end
+    end
+  end
+
   test "start" do
     SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("999") # old version
 
@@ -244,9 +257,11 @@ class CliAppTest < CliTestCase
 
   test "remove" do
     run_command("remove").tap do |output|
-      assert_match /#{Regexp.escape("sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop")}/, output
-      assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output
-      assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output
+      assert_match "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop", output
+      assert_match "docker container prune --force --filter label=service=app", output
+      assert_match "docker image prune --all --force --filter label=service=app", output
+      assert_match "rm -r .kamal/apps/app on 1.1.1.1", output
+      assert_match "rm -r .kamal/proxy/apps-config/app on 1.1.1.1", output
     end
   end
 
@@ -268,6 +283,13 @@ class CliAppTest < CliTestCase
     end
   end
 
+  test "remove_app_directories" do
+    run_command("remove_app_directories").tap do |output|
+      assert_match "rm -r .kamal/apps/app on 1.1.1.1", output
+      assert_match "rm -r .kamal/proxy/apps-config/app on 1.1.1.1", output
+    end
+  end
+
   test "exec" do
     run_command("exec", "ruby -v").tap do |output|
       assert_match "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:latest ruby -v", output
@@ -437,6 +459,24 @@ class CliAppTest < CliTestCase
     end
   end
 
+  test "live" do
+    run_command("live").tap do |output|
+      assert_match "docker exec kamal-proxy kamal-proxy resume app-web on 1.1.1.1", output
+    end
+  end
+
+  test "maintenance" do
+    run_command("maintenance").tap do |output|
+      assert_match "docker exec kamal-proxy kamal-proxy stop app-web --drain-timeout=\"30s\" on 1.1.1.1", output
+    end
+  end
+
+  test "maintenance with options" do
+    run_command("maintenance", "--message", "Hello", "--drain_timeout", "10").tap do |output|
+      assert_match "docker exec kamal-proxy kamal-proxy stop app-web --drain-timeout=\"10s\" --message=\"Hello\" on 1.1.1.1", output
+    end
+  end
+
   private
     def run_command(*command, config: :with_accessories, host: "1.1.1.1", allow_execute_error: false)
       stdouted do

test/cli/proxy_test.rb

@@ -56,12 +56,12 @@ class CliProxyTest < CliTestCase
     run_command("reboot", "-y").tap do |output|
       assert_match "docker container stop kamal-proxy on 1.1.1.1", output
       assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.1", output
-      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/app-config:/home/kamal-proxy/.app-config on 1.1.1.1", output
+      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config on 1.1.1.1", output
       assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target=\"abcdefabcdef:80\" --deploy-timeout=\"6s\" --drain-timeout=\"30s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\" on 1.1.1.1", output
 
       assert_match "docker container stop kamal-proxy on 1.1.1.2", output
       assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.2", output
-      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/app-config:/home/kamal-proxy/.app-config on 1.1.1.2", output
+      assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config on 1.1.1.2", output
       assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target=\"abcdefabcdef:80\" --deploy-timeout=\"6s\" --drain-timeout=\"30s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\" on 1.1.1.2", output
     end
   end

test/commands/app_test.rb

@@ -497,6 +497,30 @@ class CommandsAppTest < ActiveSupport::TestCase
     ], new_command(asset_path: "/public/assets").clean_up_assets
   end
 
+  test "live" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy resume app-web",
+      new_command.live.join(" ")
+  end
+
+  test "maintenance" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy stop app-web",
+      new_command.maintenance.join(" ")
+  end
+
+  test "maintenance with options" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy stop app-web --drain-timeout=\"10s\" --message=\"Hi\"",
+      new_command.maintenance(drain_timeout: 10, message: "Hi").join(" ")
+  end
+
+  test "remove_proxy_app_directory" do
+    assert_equal \
+      "rm -r .kamal/proxy/apps-config/app",
+      new_command.remove_proxy_app_directory.join(" ")
+  end
+
   private
     def new_command(role: "web", host: "1.1.1.1", **additional_config)
       config = Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999")

test/commands/proxy_test.rb

@@ -15,7 +15,7 @@ class CommandsProxyTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/app-config:/home/kamal-proxy/.app-config",
+      "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config",
       new_command.run.join(" ")
   end
 
@@ -23,7 +23,7 @@ class CommandsProxyTest < ActiveSupport::TestCase
     @config.delete(:proxy)
 
     assert_equal \
-      "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/app-config:/home/kamal-proxy/.app-config",
+      "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .kamal/proxy/image 2> /dev/null || echo \"basecamp/kamal-proxy\"):$(cat .kamal/proxy/image_version 2> /dev/null || echo \"#{Kamal::Configuration::PROXY_MINIMUM_VERSION}\") | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy --volume $(pwd)/.kamal/proxy/apps-config:/home/kamal-proxy/.apps-config",
       new_command.run.join(" ")
   end
 

test/configuration_test.rb

@@ -395,4 +395,13 @@ class ConfigurationTest < ActiveSupport::TestCase
 
     assert_equal "Different roles can't share the same host for SSL: foo.example.com", exception.message
   end
+
+  test "proxy directories" do
+    assert_equal ".kamal/proxy/apps-config", @config.proxy_apps_directory
+    assert_equal "/home/kamal-proxy/.apps-config", @config.proxy_apps_container_directory
+    assert_equal ".kamal/proxy/apps-config/app", @config.proxy_app_directory
+    assert_equal "/home/kamal-proxy/.apps-config/app", @config.proxy_app_container_directory
+    assert_equal ".kamal/proxy/apps-config/app/error_pages", @config.proxy_error_pages_directory
+    assert_equal "/home/kamal-proxy/.apps-config/app/error_pages", @config.proxy_error_pages_container_directory
+  end
 end

test/fixtures/deploy_with_error_pages.yml

@@ -0,0 +1,11 @@
+service: app
+image: dhh/app
+servers:
+  - "1.1.1.1"
+  - "1.1.1.2"
+registry:
+  username: user
+  password: pw
+builder:
+  arch: amd64
+error_pages_path: public

test/integration/app_test.rb

@@ -48,9 +48,36 @@ class AppTest < IntegrationTest
     assert_match "App Host: vm1", exec_output
     assert_match /1 root      0:\d\d nginx/, exec_output
 
+    kamal :app, :maintenance
+    assert_app_in_maintenance
+
+    kamal :app, :live
+    assert_app_is_up
+
     kamal :app, :remove
 
     assert_app_not_found
     assert_app_directory_removed
   end
+
+  test "custom error pages" do
+    @app = "app_with_roles"
+
+    kamal :deploy
+    assert_app_is_up
+
+    kamal :app, :maintenance
+    assert_app_in_maintenance message: "Custom Maintenance Page"
+
+    kamal :app, :live
+    kamal :app, :maintenance, "--message", "\"Testing Maintence Mode\""
+    assert_app_in_maintenance message: "Custom Maintenance Page: Testing Maintence Mode"
+
+    second_version = update_app_rev
+
+    kamal :redeploy
+
+    kamal :app, :maintenance
+    assert_app_in_maintenance message: "Custom Maintenance Page"
+  end
 end

test/integration/docker/deployer/app_with_roles/config/deploy.yml

@@ -37,6 +37,7 @@ proxy:
       - X-Request-Start
 
 asset_path: /usr/share/nginx/html/versions
+error_pages_path: error_pages
 
 registry:
   server: registry:4443

test/integration/docker/deployer/app_with_roles/error_pages/503.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>503 Service Interrupted</title>
+  </head>
+  <body>
+    <p>Custom Maintenance Page: {{ .Message }}</p>
+  </body>
+</html>

test/integration/integration_test.rb

@@ -45,15 +45,22 @@ class IntegrationTest < ActiveSupport::TestCase
     end
 
     def assert_app_is_down
-      response = app_response
-      debug_response_code(response, "502")
-      assert_equal "502", response.code
+      assert_app_error_code("502")
+    end
+
+    def assert_app_in_maintenance(message: nil)
+      assert_app_error_code("503", message: message)
     end
 
     def assert_app_not_found
+      assert_app_error_code("404")
+    end
+
+    def assert_app_error_code(code, message: nil)
       response = app_response
-      debug_response_code(response, "404")
-      assert_equal "404", response.code
+      debug_response_code(response, code)
+      assert_equal code, response.code
+      assert_match message, response.body.strip if message
     end
 
     def assert_app_is_up(version: nil, app: @app)

test/test_helper.rb

@@ -58,9 +58,7 @@ class ActiveSupport::TestCase
     def setup_test_secrets(**files)
       @original_pwd = Dir.pwd
       @secrets_tmpdir = Dir.mktmpdir
-      fixtures_dup = File.join(@secrets_tmpdir, "test")
-      FileUtils.mkdir_p(fixtures_dup)
-      FileUtils.cp_r("test/fixtures/", fixtures_dup)
+      copy_fixtures(@secrets_tmpdir)
 
       Dir.chdir(@secrets_tmpdir)
       FileUtils.mkdir_p(".kamal")
@@ -75,6 +73,30 @@ class ActiveSupport::TestCase
       Dir.chdir(@original_pwd)
       FileUtils.rm_rf(@secrets_tmpdir)
     end
+
+    def with_error_pages(directory:)
+      error_pages_tmpdir = Dir.mktmpdir
+
+      Dir.mktmpdir do |tmpdir|
+        copy_fixtures(tmpdir)
+
+        Dir.chdir(tmpdir) do
+          FileUtils.mkdir_p(directory)
+          Dir.chdir(directory) do
+            File.write("404.html", "404 page")
+            File.write("503.html", "503 page")
+          end
+
+          yield
+        end
+      end
+    end
+
+    def copy_fixtures(to_dir)
+      new_test_dir = File.join(to_dir, "test")
+      FileUtils.mkdir_p(new_test_dir)
+      FileUtils.cp_r("test/fixtures/", new_test_dir)
+    end
 end
 
 class SecretAdapterTestCase < ActiveSupport::TestCase