Add pre and post app boot hooks

Add two new hooks pre-app-boot and post-app-boot. They are analagous
to the pre/post proxy reboot hooks.

If the boot strategy deploys in groups, then the hooks are called once
per group of hosts and `KAMAL_HOSTS` contains a comma delimited list of
the hosts in that group.

If all hosts are deployed to at once, then they are called once with
`KAMAL_HOSTS` containing all the hosts.

It is possible to have pauses between groups of hosts in the boot config,
where this is the case the pause happens after the post-app-boot hook is
called.

lib/kamal/cli/app.rb

@@ -16,12 +16,20 @@ class Kamal::Cli::App < Kamal::Cli::Base
         # Primary hosts and roles are returned first, so they can open the barrier
         barrier = Kamal::Cli::Healthcheck::Barrier.new
 
-        on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
+        host_boot_groups.each do |hosts|
+          host_list = Array(hosts).join(",")
+          run_hook "pre-app-boot", hosts: host_list
+
+          on(hosts) do |host|
             KAMAL.roles_on(host).each do |role|
               Kamal::Cli::App::Boot.new(host, role, self, version, barrier).run
             end
           end
 
+          run_hook "post-app-boot", hosts: host_list
+          sleep KAMAL.config.boot.wait if KAMAL.config.boot.wait
+        end
+
         # Tag once the app booted on all hosts
         on(KAMAL.hosts) do |host|
           execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
@@ -340,4 +348,8 @@ class Kamal::Cli::App < Kamal::Cli::Base
         yield
       end
     end
+
+    def host_boot_groups
+      KAMAL.config.boot.limit ? KAMAL.hosts.each_slice(KAMAL.config.boot.limit).to_a : [ KAMAL.hosts ]
+    end
 end

lib/kamal/cli/templates/sample_hooks/post-app-boot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Booted app version $KAMAL_VERSION on $KAMAL_HOSTS..."

lib/kamal/cli/templates/sample_hooks/pre-app-boot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Booting app version $KAMAL_VERSION on $KAMAL_HOSTS..."

lib/kamal/commander.rb

@@ -136,14 +136,6 @@ class Kamal::Commander
     SSHKit.config.output_verbosity = old_level
   end
 
-  def boot_strategy
-    if config.boot.limit.present?
-      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
-    else
-      {}
-    end
-  end
-
   def holding_lock?
     self.holding_lock
   end

test/cli/app_test.rb

@@ -37,13 +37,20 @@ class CliAppTest < CliTestCase
   end
 
   test "boot uses group strategy when specified" do
-    Kamal::Cli::App.any_instance.stubs(:on).with("1.1.1.1").times(2) # ensure locks dir, acquire & release lock
+    Kamal::Cli::App.any_instance.stubs(:on).with("1.1.1.1").twice
-    Kamal::Cli::App.any_instance.stubs(:on).with([ "1.1.1.1" ]) # tag container
+    Kamal::Cli::App.any_instance.stubs(:on).with([ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ]).times(3)
 
     # Strategy is used when booting the containers
-    Kamal::Cli::App.any_instance.expects(:on).with([ "1.1.1.1" ], in: :groups, limit: 3, wait: 2).with_block_given
+    Kamal::Cli::App.any_instance.expects(:on).with([ "1.1.1.1", "1.1.1.2", "1.1.1.3" ]).with_block_given
+    Kamal::Cli::App.any_instance.expects(:on).with([ "1.1.1.4" ]).with_block_given
+    Object.any_instance.expects(:sleep).with(2).twice
 
-    run_command("boot", config: :with_boot_strategy)
+    Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+
+    run_command("boot", config: :with_boot_strategy, host: nil).tap do |output|
+      assert_hook_ran "pre-app-boot", output, count: 2
+      assert_hook_ran "post-app-boot", output, count: 2
+    end
   end
 
   test "boot errors don't leave lock in place" do

test/cli/build_test.rb

@@ -11,7 +11,6 @@ class CliBuildTest < CliTestCase
   test "push" do
     with_build_directory do |build_directory|
       Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
-      hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" }
 
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
         .with(:git, "-C", anything, :"rev-parse", :HEAD)
@@ -22,7 +21,7 @@ class CliBuildTest < CliTestCase
         .returns("")
 
       run_command("push", "--verbose").tap do |output|
-        assert_hook_ran "pre-build", output, **hook_variables
+        assert_hook_ran "pre-build", output
         assert_match /Cloning repo into build directory/, output
         assert_match /git -C #{Dir.tmpdir}\/kamal-clones\/app-#{pwd_sha} clone #{Dir.pwd}/, output
         assert_match /docker --version && docker buildx version/, output
@@ -34,7 +33,6 @@ class CliBuildTest < CliTestCase
   test "push --output=docker" do
     with_build_directory do |build_directory|
       Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
-      hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" }
 
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
         .with(:git, "-C", anything, :"rev-parse", :HEAD)
@@ -45,7 +43,7 @@ class CliBuildTest < CliTestCase
         .returns("")
 
       run_command("push", "--output=docker", "--verbose").tap do |output|
-        assert_hook_ran "pre-build", output, **hook_variables
+        assert_hook_ran "pre-build", output
         assert_match /Cloning repo into build directory/, output
         assert_match /git -C #{Dir.tmpdir}\/kamal-clones\/app-#{pwd_sha} clone #{Dir.pwd}/, output
         assert_match /docker --version && docker buildx version/, output
@@ -91,11 +89,10 @@ class CliBuildTest < CliTestCase
 
   test "push without clone" do
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
-    hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" }
 
     run_command("push", "--verbose", fixture: :without_clone).tap do |output|
       assert_no_match /Cloning repo into build directory/, output
-      assert_hook_ran "pre-build", output, **hook_variables
+      assert_hook_ran "pre-build", output
       assert_match /docker --version && docker buildx version/, output
       assert_match /docker buildx build --output=type=registry --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile . as .*@localhost/, output
     end

test/cli/cli_test_case.rb

@@ -40,8 +40,9 @@ class CliTestCase < ActiveSupport::TestCase
         .with(:docker, :buildx, :inspect, "kamal-local-docker-container")
     end
 
-    def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: false, secrets: false)
+    def assert_hook_ran(hook, output, count: 1)
-      assert_match %r{usr/bin/env\s\.kamal/hooks/#{hook}}, output
+      regexp = ([ "/usr/bin/env .kamal/hooks/#{hook}" ] * count).join(".*")
+      assert_match /#{regexp}/m, output
     end
 
     def with_argv(*argv)

test/cli/main_test.rb

@@ -53,17 +53,16 @@ class CliMainTest < CliTestCase
       Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
       Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
-      hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "deploy" }
 
       run_command("deploy", "--verbose").tap do |output|
-        assert_hook_ran "pre-connect", output, **hook_variables
+        assert_hook_ran "pre-connect", output
         assert_match /Log into image registry/, output
         assert_match /Build and push app image/, output
-        assert_hook_ran "pre-deploy", output, **hook_variables, secrets: true
+        assert_hook_ran "pre-deploy", output
         assert_match /Ensure kamal-proxy is running/, output
         assert_match /Detect stale containers/, output
         assert_match /Prune old containers and images/, output
-        assert_hook_ran "post-deploy", output, **hook_variables, runtime: true, secrets: true
+        assert_hook_ran "post-deploy", output
       end
     end
   end
@@ -205,14 +204,12 @@ class CliMainTest < CliTestCase
 
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
 
-    hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "redeploy" }
-
     run_command("redeploy", "--verbose").tap do |output|
-      assert_hook_ran "pre-connect", output, **hook_variables
+      assert_hook_ran "pre-connect", output
       assert_match /Build and push app image/, output
-      assert_hook_ran "pre-deploy", output, **hook_variables
+      assert_hook_ran "pre-deploy", output
       assert_match /Running the pre-deploy hook.../, output
-      assert_hook_ran "post-deploy", output, **hook_variables, runtime: true
+      assert_hook_ran "post-deploy", output
     end
   end
 
@@ -258,14 +255,13 @@ class CliMainTest < CliTestCase
       .returns("running").at_least_once # health check
 
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
-    hook_variables = { version: 123, service_version: "app@123", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "rollback" }
 
     run_command("rollback", "--verbose", "123", config_file: "deploy_with_accessories").tap do |output|
-      assert_hook_ran "pre-deploy", output, **hook_variables
+      assert_hook_ran "pre-deploy", output
       assert_match "docker tag dhh/app:123 dhh/app:latest", output
       assert_match "docker run --detach --restart unless-stopped --name app-web-123", output
       assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running"
-      assert_hook_ran "post-deploy", output, **hook_variables, runtime: true
+      assert_hook_ran "post-deploy", output
     end
   end
 

test/commander_test.rb

@@ -104,28 +104,6 @@ class CommanderTest < ActiveSupport::TestCase
     assert_equal [ "web", "workers" ], @kamal.roles_on("1.1.1.1").map(&:name)
   end
 
-  test "default group strategy" do
-    assert_empty @kamal.boot_strategy
-  end
-
-  test "specific limit group strategy" do
-    configure_with(:deploy_with_boot_strategy)
-
-    assert_equal({ in: :groups, limit: 3, wait: 2 }, @kamal.boot_strategy)
-  end
-
-  test "percentage-based group strategy" do
-    configure_with(:deploy_with_percentage_boot_strategy)
-
-    assert_equal({ in: :groups, limit: 1, wait: 2 }, @kamal.boot_strategy)
-  end
-
-  test "percentage-based group strategy limit is at least 1" do
-    configure_with(:deploy_with_low_percentage_boot_strategy)
-
-    assert_equal({ in: :groups, limit: 1, wait: 2 }, @kamal.boot_strategy)
-  end
-
   test "try to match the primary role from a list of specific roles" do
     configure_with(:deploy_primary_web_role_override)
 

test/configuration/boot_test.rb

@@ -0,0 +1,54 @@
+require "test_helper"
+
+class ConfigurationBootTest < ActiveSupport::TestCase
+  test "no group strategy" do
+    deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, builder: { "arch" => "amd64" },
+      servers: { "web" => [ "1.1.1.1", "1.1.1.2" ], "workers" => [ "1.1.1.3", "1.1.1.4" ] }
+    }
+
+    config = Kamal::Configuration.new(deploy)
+
+    assert_nil config.boot.limit
+    assert_nil config.boot.wait
+  end
+
+  test "specific limit group strategy" do
+    deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, builder: { "arch" => "amd64" },
+      servers: { "web" => [ "1.1.1.1", "1.1.1.2" ], "workers" => [ "1.1.1.3", "1.1.1.4" ] },
+      boot: { "limit" => 3, "wait" => 2 }
+    }
+
+    config = Kamal::Configuration.new(deploy)
+
+    assert_equal 3, config.boot.limit
+    assert_equal 2, config.boot.wait
+  end
+
+  test "percentage-based group strategy" do
+    deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, builder: { "arch" => "amd64" },
+      servers: { "web" => [ "1.1.1.1", "1.1.1.2" ], "workers" => [ "1.1.1.3", "1.1.1.4" ] },
+      boot: { "limit" => "50%", "wait" => 2 }
+    }
+
+    config = Kamal::Configuration.new(deploy)
+
+    assert_equal 2, config.boot.limit
+    assert_equal 2, config.boot.wait
+  end
+
+  test "percentage-based group strategy limit is at least 1" do
+    deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, builder: { "arch" => "amd64" },
+      servers: { "web" => [ "1.1.1.1", "1.1.1.2" ], "workers" => [ "1.1.1.3", "1.1.1.4" ] },
+      boot: { "limit" => "1%", "wait" => 2 }
+    }
+
+    config = Kamal::Configuration.new(deploy)
+
+    assert_equal 1, config.boot.limit
+    assert_equal 2, config.boot.wait
+  end
+end

test/fixtures/deploy_with_low_percentage_boot_strategy.yml

@@ -1,19 +0,0 @@
-service: app
-image: dhh/app
-servers:
-  web:
-    - "1.1.1.1"
-    - "1.1.1.2"
-  workers:
-    - "1.1.1.3"
-    - "1.1.1.4"
-builder:
-  arch: amd64
-
-registry:
-  username: user
-  password: pw
-
-boot:
-  limit: 1%
-  wait: 2

test/fixtures/deploy_with_percentage_boot_strategy.yml

@@ -1,19 +0,0 @@
-service: app
-image: dhh/app
-servers:
-  web:
-    - "1.1.1.1"
-    - "1.1.1.2"
-  workers:
-    - "1.1.1.3"
-    - "1.1.1.4"
-builder:
-  arch: amd64
-
-registry:
-  username: user
-  password: pw
-
-boot:
-  limit: 1%
-  wait: 2

test/integration/app_test.rb

@@ -15,7 +15,9 @@ class AppTest < IntegrationTest
     # kamal app start does not wait
     wait_for_app_to_be_up
 
-    kamal :app, :boot
+    output = kamal :app, :boot, "--verbose", capture: true
+    assert_match "Booting app on vm1,vm2...", output
+    assert_match "Booted app on vm1,vm2...", output
 
     wait_for_app_to_be_up
 

test/integration/docker/deployer/app/.kamal/hooks/post-app-boot

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "Booted app on ${KAMAL_HOSTS}..."
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/post-app-boot

test/integration/docker/deployer/app/.kamal/hooks/pre-app-boot

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "Booting app on ${KAMAL_HOSTS}..."
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-app-boot

test/integration/main_test.rb

@@ -8,19 +8,19 @@ class MainTest < IntegrationTest
 
     kamal :deploy
     assert_app_is_up version: first_version
-    assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy"
+    assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "pre-app-boot", "post-app-boot", "post-deploy"
     assert_envs version: first_version
 
     second_version = update_app_rev
 
     kamal :redeploy
     assert_app_is_up version: second_version
-    assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy"
+    assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "pre-app-boot", "post-app-boot", "post-deploy"
 
     assert_accumulated_assets first_version, second_version
 
     kamal :rollback, first_version
-    assert_hooks_ran "pre-connect", "pre-deploy", "post-deploy"
+    assert_hooks_ran "pre-connect", "pre-deploy", "pre-app-boot", "post-app-boot", "post-deploy"
     assert_app_is_up version: first_version
 
     details = kamal :details, capture: true

test/test_helper.rb

@@ -36,6 +36,10 @@ class ActiveSupport::TestCase
   extend Rails::LineFiltering
 
   private
+    setup do
+      SSHKit::Backend::Netssh.pool.close_connections
+    end
+
     def stdouted
       capture(:stdout) { yield }.strip
     end