From db0bf6bb16efdeb1c4f60558802554fdfbf79f65 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Mon, 29 May 2023 15:37:53 +0100 Subject: [PATCH] Add a pre-deploy hook Useful for checking the status of CI before deploying. Doing this at this point in the deployment maximises the parallelisation of building and running CI. --- README.md | 9 +- lib/mrsk/cli/base.rb | 30 ++++++- lib/mrsk/cli/main.rb | 6 ++ .../templates/sample_hooks/pre-deploy.sample | 82 +++++++++++++++++++ test/cli/build_test.rb | 4 + test/cli/cli_test_case.rb | 29 +++++-- test/cli/main_test.rb | 17 +++- .../deployer/app/.mrsk/hooks/pre-deploy | 3 + test/integration/main_test.rb | 6 +- 9 files changed, 165 insertions(+), 21 deletions(-) create mode 100755 lib/mrsk/cli/templates/sample_hooks/pre-deploy.sample create mode 100755 test/integration/docker/deployer/app/.mrsk/hooks/pre-deploy diff --git a/README.md b/README.md index ca9bdb8e..81f9deb9 100644 --- a/README.md +++ b/README.md @@ -882,11 +882,13 @@ firing a JSON webhook. These variables include: - `MRSK_PERFORMER` - the local user performing the command (from `whoami`) - `MRSK_SERVICE_VERSION` - an abbreviated service and version for use in messages, e.g. app@150b24f - `MRSK_VERSION` - an full version being deployed -- `MRSK_DESTINATION` - optional: destination, e.g. "staging" - `MRSK_HOSTS` - a comma separated list of the hosts targeted by the command +- `MRSK_COMMAND` - The command we are running +- `MRSK_SUBCOMMAND` - optional: The subcommand we are running +- `MRSK_DESTINATION` - optional: destination, e.g. "staging" - `MRSK_ROLE` - optional: role targeted, e.g. "web" -There are three hooks: +There are four hooks: 1. pre-connect Called before taking the deploy lock. For checks that need to run before connecting to remote hosts - e.g. DNS warming. @@ -894,6 +896,9 @@ Called before taking the deploy lock. For checks that need to run before connect 2. pre-build Used for pre-build checks - e.g. there are no uncommitted changes or that CI has passed. +3. pre-deploy +For final checks before deploying, e.g. checking CI completed + 3. post-deploy - run after a deploy, redeploy or rollback This hook is also passed a `MRSK_RUNTIME` env variable. diff --git a/lib/mrsk/cli/base.rb b/lib/mrsk/cli/base.rb index 439223c4..8834ec11 100644 --- a/lib/mrsk/cli/base.rb +++ b/lib/mrsk/cli/base.rb @@ -133,15 +133,39 @@ module Mrsk::Cli end end - def run_hook(hook, **details) + def run_hook(hook, **extra_details) if !options[:skip_hooks] && MRSK.hook.hook_exists?(hook) + details = { hosts: MRSK.hosts.join(","), command: command, subcommand: subcommand } + say "Running the #{hook} hook...", :magenta run_locally do - MRSK.with_verbosity(:debug) { execute *MRSK.hook.run(hook, **details, hosts: MRSK.hosts.join(",")) } + MRSK.with_verbosity(:debug) { execute *MRSK.hook.run(hook, **details, **extra_details) } rescue SSHKit::Command::Failed raise HookError.new("Hook `#{hook}` failed") end end end - end + + def command + @mrsk_command ||= begin + invocation_class, invocation_commands = *first_invocation + if invocation_class == Mrsk::Cli::Main + invocation_commands[0] + else + Mrsk::Cli::Main.subcommand_classes.find { |command, clazz| clazz == invocation_class }[0] + end + end + end + + def subcommand + @mrsk_subcommand ||= begin + invocation_class, invocation_commands = *first_invocation + invocation_commands[0] if invocation_class != Mrsk::Cli::Main + end + end + + def first_invocation + instance_variable_get("@_invocations").first + end + end end diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index 8f078bdc..18371bbf 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -28,6 +28,8 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base invoke "mrsk:cli:build:deliver", [], invoke_options end + run_hook "pre-deploy" + say "Ensure Traefik is running...", :magenta invoke "mrsk:cli:traefik:boot", [], invoke_options @@ -62,6 +64,8 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base invoke "mrsk:cli:build:deliver", [], invoke_options end + run_hook "pre-deploy" + say "Ensure app can pass healthcheck...", :magenta invoke "mrsk:cli:healthcheck:perform", [], invoke_options @@ -86,6 +90,8 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base old_version = nil if container_available?(version) + run_hook "pre-deploy" + invoke "mrsk:cli:app:boot", [], invoke_options.merge(version: version) rolled_back = true else diff --git a/lib/mrsk/cli/templates/sample_hooks/pre-deploy.sample b/lib/mrsk/cli/templates/sample_hooks/pre-deploy.sample new file mode 100755 index 00000000..061d2bdf --- /dev/null +++ b/lib/mrsk/cli/templates/sample_hooks/pre-deploy.sample @@ -0,0 +1,82 @@ +#!/bin/sh + +# A sample pre-deploy hook +# +# Checks the Github status of the build, waiting for a pending build to complete for up to 720 seconds. +# +# Fails unless the combined status is "success" +# +# These environment variables are available: +# MRSK_RECORDED_AT +# MRSK_PERFORMER +# MRSK_VERSION +# MRSK_HOSTS +# MRSK_COMMAND +# MRSK_SUBCOMMAND +# MRSK_ROLE (if set) +# MRSK_DESTINATION (if set) + +#!/usr/bin/env ruby + +# Only check the build status for production deployments +if ENV["MRSK_COMMAND"] == "rollback" || ENV["MRSK_DESTINATION"] != "production" + exit 0 +end + +require "bundler/inline" + +# true = install gems so this is fast on repeat invocations +gemfile(true, quiet: true) do + source "https://rubygems.org" + + gem "octokit" + gem "faraday-retry" +end + +MAX_ATTEMPTS = 72 +ATTEMPTS_GAP = 10 + +def exit_with_error(message) + $stderr.puts message + exit 1 +end + +def first_status_url(combined_status, state) + first_status = combined_status[:statuses].find { |status| status[:state] == state } + first_status && first_status[:target_url] +end + +remote_url = `git config --get remote.origin.url`.strip.delete_prefix("https://github.com/") +git_sha = `git rev-parse HEAD`.strip + +repository = Octokit::Repository.from_url(remote_url) +github_client = Octokit::Client.new(access_token: ENV["GITHUB_TOKEN"]) +attempts = 0 + +begin + loop do + combined_status = github_client.combined_status(remote_url, git_sha) + state = combined_status[:state] + first_status_url = first_status_url(combined_status, state) + + case state + when "success" + puts "Build passed, see #{first_status_url}" + exit 0 + when "failure" + exit_with_error "Build failed, see #{first_status_url}" + when "pending" + attempts += 1 + end + + puts "Waiting #{ATTEMPTS_GAP} more seconds for build to complete#{", see #{first_status_url}" if first_status_url}..." + + if attempts == MAX_ATTEMPTS + exit_with_error "Build status is still pending, gave up after #{MAX_ATTEMPTS * ATTEMPTS_GAP} seconds" + end + + sleep(ATTEMPTS_GAP) + end +rescue Octokit::NotFound + exit_with_error "Build status could not be found" +end diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb index ece83851..68b20509 100644 --- a/test/cli/build_test.rb +++ b/test/cli/build_test.rb @@ -9,7 +9,11 @@ class CliBuildTest < CliTestCase end test "push" do + Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) + hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "build", subcommand: "push" } + run_command("push").tap do |output| + assert_hook_ran "pre-build", output, **hook_variables assert_match /docker --version && docker buildx version/, output assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder mrsk-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output end diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb index 2455002e..05b0ff9f 100644 --- a/test/cli/cli_test_case.rb +++ b/test/cli/cli_test_case.rb @@ -27,19 +27,30 @@ class CliTestCase < ActiveSupport::TestCase .raises(SSHKit::Command::Failed.new("failed")) end - def ensure_hook_runs(hook) - Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) - SSHKit::Backend::Abstract.any_instance.stubs(:execute) - .with { |*args| args != [".mrsk/hooks/#{hook}"] } - SSHKit::Backend::Abstract.any_instance.expects(:execute) - .with { |*args| args.first == ".mrsk/hooks/#{hook}" } - .once - end - def stub_locking SSHKit::Backend::Abstract.any_instance.stubs(:execute) .with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock } SSHKit::Backend::Abstract.any_instance.stubs(:execute) .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" } end + + def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: nil) + performer = `whoami`.strip + + assert_match "Running the #{hook} hook...\n", output + + expected = %r{Running\s/usr/bin/env\s\.mrsk/hooks/#{hook}\sas\s#{performer}@localhost\n\s + DEBUG\s\[[0-9a-f]*\]\sCommand:\s\(\sexport\s + MRSK_RECORDED_AT=\"\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ\"\s + MRSK_PERFORMER=\"#{performer}\"\s + MRSK_VERSION=\"#{version}\"\s + MRSK_SERVICE_VERSION=\"#{service_version}\"\s + MRSK_HOSTS=\"#{hosts}\"\s + MRSK_COMMAND=\"#{command}\"\s + #{"MRSK_SUBCOMMAND=\\\"#{subcommand}\\\"\\s" if subcommand} + #{"MRSK_RUNTIME=\\\"#{runtime}\\\"\\s" if runtime} + ;\s/usr/bin/env\s\.mrsk/hooks/#{hook} }x + + assert_match expected, output + end end diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index cdbcc9fa..a5ff9d93 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -21,16 +21,18 @@ class CliMainTest < CliTestCase Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options) Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) + hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "deploy" } run_command("deploy").tap do |output| - assert_match /Running the pre-connect hook.../, output + assert_hook_ran "pre-connect", output, **hook_variables assert_match /Log into image registry/, output assert_match /Build and push app image/, output + assert_hook_ran "pre-deploy", output, **hook_variables assert_match /Ensure Traefik is running/, output assert_match /Ensure app can pass healthcheck/, output assert_match /Detect stale containers/, output assert_match /Prune old containers and images/, output - assert_match /Running the post-deploy hook.../, output + assert_hook_ran "post-deploy", output, **hook_variables, runtime: 0 end end @@ -124,10 +126,15 @@ class CliMainTest < CliTestCase Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) + hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "redeploy" } + run_command("redeploy").tap do |output| + assert_hook_ran "pre-connect", output, **hook_variables assert_match /Build and push app image/, output + assert_hook_ran "pre-deploy", output, **hook_variables + assert_match /Running the pre-deploy hook.../, output assert_match /Ensure app can pass healthcheck/, output - assert_match /Running the post-deploy hook.../, output + assert_hook_ran "post-deploy", output, **hook_variables, runtime: "0" end end @@ -173,13 +180,15 @@ class CliMainTest < CliTestCase end Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) + hook_variables = { version: 123, service_version: "app@123", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "rollback" } run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output| assert_match "Start container with version 123", output + assert_hook_ran "pre-deploy", output, **hook_variables assert_match "docker tag dhh/app:123 dhh/app:latest", output assert_match "docker start app-web-123", output assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running" - assert_match "Running the post-deploy hook...", output + assert_hook_ran "post-deploy", output, **hook_variables, runtime: "0" end end diff --git a/test/integration/docker/deployer/app/.mrsk/hooks/pre-deploy b/test/integration/docker/deployer/app/.mrsk/hooks/pre-deploy new file mode 100755 index 00000000..32fa04c6 --- /dev/null +++ b/test/integration/docker/deployer/app/.mrsk/hooks/pre-deploy @@ -0,0 +1,3 @@ +#!/bin/sh +echo "Deployed!" +mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-deploy diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb index 341af5eb..cc7f8bbb 100644 --- a/test/integration/main_test.rb +++ b/test/integration/main_test.rb @@ -8,16 +8,16 @@ class MainTest < IntegrationTest mrsk :deploy assert_app_is_up version: first_version - assert_hooks_ran "pre-connect", "pre-build", "post-deploy" + assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy" second_version = update_app_rev mrsk :redeploy assert_app_is_up version: second_version - assert_hooks_ran "pre-connect", "pre-build", "post-deploy" + assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy" mrsk :rollback, first_version - assert_hooks_ran "pre-connect", "post-deploy" + assert_hooks_ran "pre-connect", "pre-deploy", "post-deploy" assert_app_is_up version: first_version details = mrsk :details, capture: true