From cd73cea850363afc6b70914886f602f6c32a9017 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Mon, 3 Feb 2025 14:14:39 +0000 Subject: [PATCH] Add pre and post app boot hooks Add two new hooks pre-app-boot and post-app-boot. They are analagous to the pre/post proxy reboot hooks. If the boot strategy deploys in groups, then the hooks are called once per group of hosts and `KAMAL_HOSTS` contains a comma delimited list of the hosts in that group. If all hosts are deployed to at once, then they are called once with `KAMAL_HOSTS` containing all the hosts. It is possible to have pauses between groups of hosts in the boot config, where this is the case the pause happens after the post-app-boot hook is called. --- lib/kamal/cli/app.rb | 18 +++++-- .../sample_hooks/post-app-boot.sample | 3 ++ .../sample_hooks/pre-app-boot.sample | 3 ++ lib/kamal/commander.rb | 8 --- test/cli/app_test.rb | 15 ++++-- test/cli/build_test.rb | 9 ++-- test/cli/cli_test_case.rb | 5 +- test/cli/main_test.rb | 20 +++---- test/commander_test.rb | 22 -------- test/configuration/boot_test.rb | 54 +++++++++++++++++++ ...ploy_with_low_percentage_boot_strategy.yml | 19 ------- .../deploy_with_percentage_boot_strategy.yml | 19 ------- test/integration/app_test.rb | 4 +- .../deployer/app/.kamal/hooks/post-app-boot | 3 ++ .../deployer/app/.kamal/hooks/pre-app-boot | 3 ++ test/integration/main_test.rb | 6 +-- test/test_helper.rb | 4 ++ 17 files changed, 116 insertions(+), 99 deletions(-) create mode 100755 lib/kamal/cli/templates/sample_hooks/post-app-boot.sample create mode 100755 lib/kamal/cli/templates/sample_hooks/pre-app-boot.sample create mode 100644 test/configuration/boot_test.rb delete mode 100644 test/fixtures/deploy_with_low_percentage_boot_strategy.yml delete mode 100644 test/fixtures/deploy_with_percentage_boot_strategy.yml create mode 100755 test/integration/docker/deployer/app/.kamal/hooks/post-app-boot create mode 100755 test/integration/docker/deployer/app/.kamal/hooks/pre-app-boot diff --git a/lib/kamal/cli/app.rb b/lib/kamal/cli/app.rb index fb665af9..553ccbb2 100644 --- a/lib/kamal/cli/app.rb +++ b/lib/kamal/cli/app.rb @@ -16,10 +16,18 @@ class Kamal::Cli::App < Kamal::Cli::Base # Primary hosts and roles are returned first, so they can open the barrier barrier = Kamal::Cli::Healthcheck::Barrier.new - on(KAMAL.hosts, **KAMAL.boot_strategy) do |host| - KAMAL.roles_on(host).each do |role| - Kamal::Cli::App::Boot.new(host, role, self, version, barrier).run + host_boot_groups.each do |hosts| + host_list = Array(hosts).join(",") + run_hook "pre-app-boot", hosts: host_list + + on(hosts) do |host| + KAMAL.roles_on(host).each do |role| + Kamal::Cli::App::Boot.new(host, role, self, version, barrier).run + end end + + run_hook "post-app-boot", hosts: host_list + sleep KAMAL.config.boot.wait if KAMAL.config.boot.wait end # Tag once the app booted on all hosts @@ -340,4 +348,8 @@ class Kamal::Cli::App < Kamal::Cli::Base yield end end + + def host_boot_groups + KAMAL.config.boot.limit ? KAMAL.hosts.each_slice(KAMAL.config.boot.limit).to_a : [ KAMAL.hosts ] + end end diff --git a/lib/kamal/cli/templates/sample_hooks/post-app-boot.sample b/lib/kamal/cli/templates/sample_hooks/post-app-boot.sample new file mode 100755 index 00000000..70f9c4bc --- /dev/null +++ b/lib/kamal/cli/templates/sample_hooks/post-app-boot.sample @@ -0,0 +1,3 @@ +#!/bin/sh + +echo "Booted app version $KAMAL_VERSION on $KAMAL_HOSTS..." diff --git a/lib/kamal/cli/templates/sample_hooks/pre-app-boot.sample b/lib/kamal/cli/templates/sample_hooks/pre-app-boot.sample new file mode 100755 index 00000000..45f73550 --- /dev/null +++ b/lib/kamal/cli/templates/sample_hooks/pre-app-boot.sample @@ -0,0 +1,3 @@ +#!/bin/sh + +echo "Booting app version $KAMAL_VERSION on $KAMAL_HOSTS..." diff --git a/lib/kamal/commander.rb b/lib/kamal/commander.rb index 6a461276..97a82ce6 100644 --- a/lib/kamal/commander.rb +++ b/lib/kamal/commander.rb @@ -136,14 +136,6 @@ class Kamal::Commander SSHKit.config.output_verbosity = old_level end - def boot_strategy - if config.boot.limit.present? - { in: :groups, limit: config.boot.limit, wait: config.boot.wait } - else - {} - end - end - def holding_lock? self.holding_lock end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index fa5049ba..19190032 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -37,13 +37,20 @@ class CliAppTest < CliTestCase end test "boot uses group strategy when specified" do - Kamal::Cli::App.any_instance.stubs(:on).with("1.1.1.1").times(2) # ensure locks dir, acquire & release lock - Kamal::Cli::App.any_instance.stubs(:on).with([ "1.1.1.1" ]) # tag container + Kamal::Cli::App.any_instance.stubs(:on).with("1.1.1.1").twice + Kamal::Cli::App.any_instance.stubs(:on).with([ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ]).times(3) # Strategy is used when booting the containers - Kamal::Cli::App.any_instance.expects(:on).with([ "1.1.1.1" ], in: :groups, limit: 3, wait: 2).with_block_given + Kamal::Cli::App.any_instance.expects(:on).with([ "1.1.1.1", "1.1.1.2", "1.1.1.3" ]).with_block_given + Kamal::Cli::App.any_instance.expects(:on).with([ "1.1.1.4" ]).with_block_given + Object.any_instance.expects(:sleep).with(2).twice - run_command("boot", config: :with_boot_strategy) + Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) + + run_command("boot", config: :with_boot_strategy, host: nil).tap do |output| + assert_hook_ran "pre-app-boot", output, count: 2 + assert_hook_ran "post-app-boot", output, count: 2 + end end test "boot errors don't leave lock in place" do diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb index f93f26ec..76db2924 100644 --- a/test/cli/build_test.rb +++ b/test/cli/build_test.rb @@ -11,7 +11,6 @@ class CliBuildTest < CliTestCase test "push" do with_build_directory do |build_directory| Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) - hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" } SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:git, "-C", anything, :"rev-parse", :HEAD) @@ -22,7 +21,7 @@ class CliBuildTest < CliTestCase .returns("") run_command("push", "--verbose").tap do |output| - assert_hook_ran "pre-build", output, **hook_variables + assert_hook_ran "pre-build", output assert_match /Cloning repo into build directory/, output assert_match /git -C #{Dir.tmpdir}\/kamal-clones\/app-#{pwd_sha} clone #{Dir.pwd}/, output assert_match /docker --version && docker buildx version/, output @@ -34,7 +33,6 @@ class CliBuildTest < CliTestCase test "push --output=docker" do with_build_directory do |build_directory| Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) - hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" } SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:git, "-C", anything, :"rev-parse", :HEAD) @@ -45,7 +43,7 @@ class CliBuildTest < CliTestCase .returns("") run_command("push", "--output=docker", "--verbose").tap do |output| - assert_hook_ran "pre-build", output, **hook_variables + assert_hook_ran "pre-build", output assert_match /Cloning repo into build directory/, output assert_match /git -C #{Dir.tmpdir}\/kamal-clones\/app-#{pwd_sha} clone #{Dir.pwd}/, output assert_match /docker --version && docker buildx version/, output @@ -91,11 +89,10 @@ class CliBuildTest < CliTestCase test "push without clone" do Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) - hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" } run_command("push", "--verbose", fixture: :without_clone).tap do |output| assert_no_match /Cloning repo into build directory/, output - assert_hook_ran "pre-build", output, **hook_variables + assert_hook_ran "pre-build", output assert_match /docker --version && docker buildx version/, output assert_match /docker buildx build --output=type=registry --platform linux\/amd64 --builder kamal-local-docker-container -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile . as .*@localhost/, output end diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb index d4b57923..b7ca9eca 100644 --- a/test/cli/cli_test_case.rb +++ b/test/cli/cli_test_case.rb @@ -40,8 +40,9 @@ class CliTestCase < ActiveSupport::TestCase .with(:docker, :buildx, :inspect, "kamal-local-docker-container") end - def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: false, secrets: false) - assert_match %r{usr/bin/env\s\.kamal/hooks/#{hook}}, output + def assert_hook_ran(hook, output, count: 1) + regexp = ([ "/usr/bin/env .kamal/hooks/#{hook}" ] * count).join(".*") + assert_match /#{regexp}/m, output end def with_argv(*argv) diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index e901c3ba..becace43 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -53,17 +53,16 @@ class CliMainTest < CliTestCase Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options) Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) - hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "deploy" } run_command("deploy", "--verbose").tap do |output| - assert_hook_ran "pre-connect", output, **hook_variables + assert_hook_ran "pre-connect", output assert_match /Log into image registry/, output assert_match /Build and push app image/, output - assert_hook_ran "pre-deploy", output, **hook_variables, secrets: true + assert_hook_ran "pre-deploy", output assert_match /Ensure kamal-proxy is running/, output assert_match /Detect stale containers/, output assert_match /Prune old containers and images/, output - assert_hook_ran "post-deploy", output, **hook_variables, runtime: true, secrets: true + assert_hook_ran "post-deploy", output end end end @@ -205,14 +204,12 @@ class CliMainTest < CliTestCase Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) - hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "redeploy" } - run_command("redeploy", "--verbose").tap do |output| - assert_hook_ran "pre-connect", output, **hook_variables + assert_hook_ran "pre-connect", output assert_match /Build and push app image/, output - assert_hook_ran "pre-deploy", output, **hook_variables + assert_hook_ran "pre-deploy", output assert_match /Running the pre-deploy hook.../, output - assert_hook_ran "post-deploy", output, **hook_variables, runtime: true + assert_hook_ran "post-deploy", output end end @@ -258,14 +255,13 @@ class CliMainTest < CliTestCase .returns("running").at_least_once # health check Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) - hook_variables = { version: 123, service_version: "app@123", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "rollback" } run_command("rollback", "--verbose", "123", config_file: "deploy_with_accessories").tap do |output| - assert_hook_ran "pre-deploy", output, **hook_variables + assert_hook_ran "pre-deploy", output assert_match "docker tag dhh/app:123 dhh/app:latest", output assert_match "docker run --detach --restart unless-stopped --name app-web-123", output assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running" - assert_hook_ran "post-deploy", output, **hook_variables, runtime: true + assert_hook_ran "post-deploy", output end end diff --git a/test/commander_test.rb b/test/commander_test.rb index 4f0a829e..4a3aa2e2 100644 --- a/test/commander_test.rb +++ b/test/commander_test.rb @@ -104,28 +104,6 @@ class CommanderTest < ActiveSupport::TestCase assert_equal [ "web", "workers" ], @kamal.roles_on("1.1.1.1").map(&:name) end - test "default group strategy" do - assert_empty @kamal.boot_strategy - end - - test "specific limit group strategy" do - configure_with(:deploy_with_boot_strategy) - - assert_equal({ in: :groups, limit: 3, wait: 2 }, @kamal.boot_strategy) - end - - test "percentage-based group strategy" do - configure_with(:deploy_with_percentage_boot_strategy) - - assert_equal({ in: :groups, limit: 1, wait: 2 }, @kamal.boot_strategy) - end - - test "percentage-based group strategy limit is at least 1" do - configure_with(:deploy_with_low_percentage_boot_strategy) - - assert_equal({ in: :groups, limit: 1, wait: 2 }, @kamal.boot_strategy) - end - test "try to match the primary role from a list of specific roles" do configure_with(:deploy_primary_web_role_override) diff --git a/test/configuration/boot_test.rb b/test/configuration/boot_test.rb new file mode 100644 index 00000000..a01e5b06 --- /dev/null +++ b/test/configuration/boot_test.rb @@ -0,0 +1,54 @@ +require "test_helper" + +class ConfigurationBootTest < ActiveSupport::TestCase + test "no group strategy" do + deploy = { + service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, builder: { "arch" => "amd64" }, + servers: { "web" => [ "1.1.1.1", "1.1.1.2" ], "workers" => [ "1.1.1.3", "1.1.1.4" ] } + } + + config = Kamal::Configuration.new(deploy) + + assert_nil config.boot.limit + assert_nil config.boot.wait + end + + test "specific limit group strategy" do + deploy = { + service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, builder: { "arch" => "amd64" }, + servers: { "web" => [ "1.1.1.1", "1.1.1.2" ], "workers" => [ "1.1.1.3", "1.1.1.4" ] }, + boot: { "limit" => 3, "wait" => 2 } + } + + config = Kamal::Configuration.new(deploy) + + assert_equal 3, config.boot.limit + assert_equal 2, config.boot.wait + end + + test "percentage-based group strategy" do + deploy = { + service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, builder: { "arch" => "amd64" }, + servers: { "web" => [ "1.1.1.1", "1.1.1.2" ], "workers" => [ "1.1.1.3", "1.1.1.4" ] }, + boot: { "limit" => "50%", "wait" => 2 } + } + + config = Kamal::Configuration.new(deploy) + + assert_equal 2, config.boot.limit + assert_equal 2, config.boot.wait + end + + test "percentage-based group strategy limit is at least 1" do + deploy = { + service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, builder: { "arch" => "amd64" }, + servers: { "web" => [ "1.1.1.1", "1.1.1.2" ], "workers" => [ "1.1.1.3", "1.1.1.4" ] }, + boot: { "limit" => "1%", "wait" => 2 } + } + + config = Kamal::Configuration.new(deploy) + + assert_equal 1, config.boot.limit + assert_equal 2, config.boot.wait + end +end diff --git a/test/fixtures/deploy_with_low_percentage_boot_strategy.yml b/test/fixtures/deploy_with_low_percentage_boot_strategy.yml deleted file mode 100644 index 8698b89f..00000000 --- a/test/fixtures/deploy_with_low_percentage_boot_strategy.yml +++ /dev/null @@ -1,19 +0,0 @@ -service: app -image: dhh/app -servers: - web: - - "1.1.1.1" - - "1.1.1.2" - workers: - - "1.1.1.3" - - "1.1.1.4" -builder: - arch: amd64 - -registry: - username: user - password: pw - -boot: - limit: 1% - wait: 2 diff --git a/test/fixtures/deploy_with_percentage_boot_strategy.yml b/test/fixtures/deploy_with_percentage_boot_strategy.yml deleted file mode 100644 index 8698b89f..00000000 --- a/test/fixtures/deploy_with_percentage_boot_strategy.yml +++ /dev/null @@ -1,19 +0,0 @@ -service: app -image: dhh/app -servers: - web: - - "1.1.1.1" - - "1.1.1.2" - workers: - - "1.1.1.3" - - "1.1.1.4" -builder: - arch: amd64 - -registry: - username: user - password: pw - -boot: - limit: 1% - wait: 2 diff --git a/test/integration/app_test.rb b/test/integration/app_test.rb index 039ef012..f128054b 100644 --- a/test/integration/app_test.rb +++ b/test/integration/app_test.rb @@ -15,7 +15,9 @@ class AppTest < IntegrationTest # kamal app start does not wait wait_for_app_to_be_up - kamal :app, :boot + output = kamal :app, :boot, "--verbose", capture: true + assert_match "Booting app on vm1,vm2...", output + assert_match "Booted app on vm1,vm2...", output wait_for_app_to_be_up diff --git a/test/integration/docker/deployer/app/.kamal/hooks/post-app-boot b/test/integration/docker/deployer/app/.kamal/hooks/post-app-boot new file mode 100755 index 00000000..c81b04da --- /dev/null +++ b/test/integration/docker/deployer/app/.kamal/hooks/post-app-boot @@ -0,0 +1,3 @@ +#!/bin/sh +echo "Booted app on ${KAMAL_HOSTS}..." +mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/post-app-boot diff --git a/test/integration/docker/deployer/app/.kamal/hooks/pre-app-boot b/test/integration/docker/deployer/app/.kamal/hooks/pre-app-boot new file mode 100755 index 00000000..0328f18a --- /dev/null +++ b/test/integration/docker/deployer/app/.kamal/hooks/pre-app-boot @@ -0,0 +1,3 @@ +#!/bin/sh +echo "Booting app on ${KAMAL_HOSTS}..." +mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-app-boot diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb index a48051fe..04040c38 100644 --- a/test/integration/main_test.rb +++ b/test/integration/main_test.rb @@ -8,19 +8,19 @@ class MainTest < IntegrationTest kamal :deploy assert_app_is_up version: first_version - assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy" + assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "pre-app-boot", "post-app-boot", "post-deploy" assert_envs version: first_version second_version = update_app_rev kamal :redeploy assert_app_is_up version: second_version - assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy" + assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "pre-app-boot", "post-app-boot", "post-deploy" assert_accumulated_assets first_version, second_version kamal :rollback, first_version - assert_hooks_ran "pre-connect", "pre-deploy", "post-deploy" + assert_hooks_ran "pre-connect", "pre-deploy", "pre-app-boot", "post-app-boot", "post-deploy" assert_app_is_up version: first_version details = kamal :details, capture: true diff --git a/test/test_helper.rb b/test/test_helper.rb index f5811872..e77a0a8a 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -36,6 +36,10 @@ class ActiveSupport::TestCase extend Rails::LineFiltering private + setup do + SSHKit::Backend::Netssh.pool.close_connections + end + def stdouted capture(:stdout) { yield }.strip end