Add post-deploy and post-rollback hooks

These replace the custom audit_broadcast_cmd code. An additional env
variable MRSK_RUNTIME is passed to them.

The audit broadcast after booting an accessory has been removed.

README.md

@@ -668,43 +668,6 @@ servers:
 
 This assumes the Cron settings are stored in `config/crontab`.
 
-### Using audit broadcasts
-
-If you'd like to broadcast audits of deploys, rollbacks, etc to a chatroom or elsewhere, you can configure the `audit_broadcast_cmd` setting with the path to a bin file that will be passed the audit line as the first argument:
-
-```yaml
-audit_broadcast_cmd:
-  bin/audit_broadcast
-```
-
-The broadcast command could look something like:
-
-```bash
-#!/usr/bin/env bash
-curl -q -d content="[My App] ${1}" https://3.basecamp.com/XXXXX/integrations/XXXXX/buckets/XXXXX/chats/XXXXX/lines
-```
-
-That'll post a line like follows to a preconfigured chatbot in Basecamp:
-
-```
-[My App] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de
-```
-
-`MRSK_*` environment variables are available to the broadcast command for
-fine-grained audit reporting, e.g. for triggering deployment reports or
-firing a JSON webhook. These variables include:
-- `MRSK_RECORDED_AT` - UTC timestamp in ISO 8601 format, e.g. `2023-04-14T17:07:31Z`
-- `MRSK_PERFORMER` - the local user performing the command (from `whoami`)
-- `MRSK_MESSAGE` - the full audit message, e.g. "Deployed app@150b24f"
-- `MRSK_DESTINATION` - optional: destination, e.g. "staging"
-- `MRSK_ROLE` - optional: role targeted, e.g. "web"
-
-Use `mrsk broadcast` to test and troubleshoot your broadcast command:
-
-```bash
-mrsk broadcast -m "test audit message"
-```
-
 ### Healthcheck
 
 MRSK uses Docker healtchecks to check the health of your application during deployment. Traefik uses this same healthcheck status to determine when a container is ready to receive traffic.
@@ -912,9 +875,38 @@ You can change their location by setting `hooks_path` in the configuration file.
 
 If the script returns a non-zero exit code the command will be aborted.
 
-There is currently one hook:
+`MRSK_*` environment variables are available to the hooks command for
+fine-grained audit reporting, e.g. for triggering deployment reports or
+firing a JSON webhook. These variables include:
+- `MRSK_RECORDED_AT` - UTC timestamp in ISO 8601 format, e.g. `2023-04-14T17:07:31Z`
+- `MRSK_PERFORMER` - the local user performing the command (from `whoami`)
+- `MRSK_MESSAGE` - the full audit message, e.g. "Deployed app@150b24f"
+- `MRSK_DESTINATION` - optional: destination, e.g. "staging"
+- `MRSK_ROLE` - optional: role targeted, e.g. "web"
 
-- pre-build
+There are three hooks:
+
+1. pre-build
+Used for pre-build checks - e.g. there are no uncommitted changes or that CI has passed.
+
+2. post-deploy and post-rollback
+
+These two hooks are also passed a `MRSK_RUNTIME` env variable.
+
+This could be used to broadcast a deployment message, or register the new version with an APM.
+
+The command could look something like:
+
+```bash
+#!/usr/bin/env bash
+curl -q -d content="[My App] ${MRSK_PERFORMER} Rolled back to version ${MRSK_VERSION}" https://3.basecamp.com/XXXXX/integrations/XXXXX/buckets/XXXXX/chats/XXXXX/lines
+```
+
+That'll post a line like follows to a preconfigured chatbot in Basecamp:
+
+```
+[My App] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de
+```
 
 ## Stage of development
 

lib/mrsk/cli/accessory.rb

@@ -14,8 +14,6 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
             execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug
             execute *accessory.run
           end
-
-          audit_broadcast "Booted accessory #{name}" unless options[:skip_broadcast]
         end
       end
     end

lib/mrsk/cli/base.rb

@@ -72,10 +72,6 @@ module Mrsk::Cli
         puts "  Finished all in #{sprintf("%.1f seconds", runtime)}"
       end
 
-      def audit_broadcast(line)
-        run_locally { execute *MRSK.auditor.broadcast(line), verbosity: :debug }
-      end
-
       def with_lock
         if MRSK.holding_lock?
           yield
@@ -135,11 +131,11 @@ module Mrsk::Cli
         end
       end
 
-      def run_hook(hook)
+      def run_hook(hook, **details)
         run_locally do
           if MRSK.hook.hook_exists?(hook)
             begin
-              MRSK.with_verbosity(:debug) { execute(*MRSK.hook.run(hook)) }
+              MRSK.with_verbosity(:debug) { execute(*MRSK.hook.run(hook, **details)) }
             rescue SSHKit::Command::Failed
               raise HookError.new("Hook `#{hook}` failed")
             end

lib/mrsk/cli/main.rb

@@ -44,7 +44,7 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       end
     end
 
-    audit_broadcast "Deployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
+    run_hook "post-deploy", runtime: runtime.round
   end
 
   desc "redeploy", "Deploy app to servers without bootstrapping servers, starting Traefik, pruning, and registry login"
@@ -72,11 +72,13 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       end
     end
 
-    audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
+    run_hook "post-deploy", runtime: runtime.round
   end
 
   desc "rollback [VERSION]", "Rollback app to VERSION"
   def rollback(version)
+    rolled_back = false
+    runtime = print_runtime do
       with_lock do
         invoke_options = deploy_options
 
@@ -86,7 +88,7 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       if container_available?(version)
         invoke "mrsk:cli:app:boot", [], invoke_options.merge(version: version)
 
-        audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast]
+        run_hook "post-deploy", runtime: runtime.round
       else
         say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
       end
@@ -180,13 +182,6 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
     end
   end
 
-  desc "broadcast", "Broadcast an audit message"
-  option :message, aliases: "-m", type: :string, desc: "Audit message", required: true
-  def broadcast
-    say "Broadcast: #{options[:message]}", :magenta
-    audit_broadcast options[:message]
-  end
-
   desc "version", "Show MRSK version"
   def version
     puts Mrsk::VERSION

lib/mrsk/cli/templates/deploy.yml

@@ -25,10 +25,6 @@ registry:
 #   secret:
 #     - RAILS_MASTER_KEY
 
-# Call a broadcast command on deploys.
-# audit_broadcast_cmd:
-#   bin/broadcast_to_bc
-
 # Use a different ssh user than root
 # ssh:
 #   user: app

lib/mrsk/cli/templates/sample_hooks/post-deploy.sample

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# A sample post-deploy hook
+#
+# Checks:
+# 1. We have a clean checkout
+# 2. A remote is configured
+# 3. The branch has been pushed to the remote
+# 4. The version we are deploying matches the remote
+#
+# These environment variables are available:
+# MRSK_RECORDED_AT
+# MRSK_PERFORMER
+# MRSK_VERSION
+# MRSK_DESTINATION (if set)
+# MRSK_RUNTIME
+
+echo "$MRSK_PERFORMER deployed $MRSK_VERSION to $MRSK_DESTINATION in $MRSK_RUNTIME seconds"

lib/mrsk/cli/templates/sample_hooks/post-rollback.sample

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# A sample post-rollback hook
+#
+# Checks:
+# 1. We have a clean checkout
+# 2. A remote is configured
+# 3. The branch has been pushed to the remote
+# 4. The version we are deploying matches the remote
+#
+# These environment variables are available:
+# MRSK_RECORDED_AT
+# MRSK_PERFORMER
+# MRSK_VERSION
+# MRSK_DESTINATION (if set)
+# MRSK_RUNTIME
+
+echo "$MRSK_PERFORMER rolled back to $MRSK_VERSION on $MRSK_DESTINATION in $MRSK_RUNTIME seconds"

lib/mrsk/commands/auditor.rb

@@ -13,14 +13,6 @@ class Mrsk::Commands::Auditor < Mrsk::Commands::Base
       audit_log_file
   end
 
-  # Runs locally
-  def broadcast(line, **details)
-    if broadcast_cmd = config.audit_broadcast_cmd
-      tags = audit_tags(**details, event: line)
-      [ broadcast_cmd,  "'#{tags.except(:recorded_at, :event, :version)} #{line}'", env: tags.env ]
-    end
-  end
-
   def reveal
     [ :tail, "-n", 50, audit_log_file ]
   end

lib/mrsk/configuration.rb

@@ -157,10 +157,6 @@ class Mrsk::Configuration
   end
 
 
-  def audit_broadcast_cmd
-    raw_config.audit_broadcast_cmd
-  end
-
   def healthcheck
     { "path" => "/up", "port" => 3000, "max_attempts" => 7 }.merge(raw_config.healthcheck || {})
   end

test/cli/build_test.rb

@@ -95,13 +95,6 @@ class CliBuildTest < CliTestCase
       stdouted { Mrsk::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
     end
 
-    def stub_locking
-      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock }
-      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" }
-    end
-
     def stub_dependency_checks
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
         .with(:docker, "--version", "&&", :docker, :buildx, "version")

test/cli/cli_test_case.rb

@@ -27,4 +27,19 @@ class CliTestCase < ActiveSupport::TestCase
         .raises(SSHKit::Command::Failed.new("failed"))
     end
 
+    def ensure_hook_runs(hook)
+      Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| args != [".mrsk/hooks/#{hook}"] }
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with { |*args| args.first == ".mrsk/hooks/#{hook}" }
+        .once
+    end
+
+    def stub_locking
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock }
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" }
+    end
 end

test/cli/main_test.rb

@@ -20,6 +20,9 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
 
+    stub_locking
+    ensure_hook_runs("post-deploy")
+
     run_command("deploy").tap do |output|
       assert_match /Log into image registry/, output
       assert_match /Build and push app image/, output
@@ -102,6 +105,9 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
 
+    stub_locking
+    ensure_hook_runs("post-deploy")
+
     run_command("redeploy").tap do |output|
       assert_match /Build and push app image/, output
       assert_match /Ensure app can pass healthcheck/, output
@@ -149,7 +155,6 @@ class CliMainTest < CliTestCase
         .returns("running").at_least_once # health check
     end
 
-
     run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
       assert_match "Start container with version 123", output
       assert_match "docker tag dhh/app:123 dhh/app:latest", output
@@ -180,6 +185,13 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "rollback runs post deploy hook" do
+    Mrsk::Cli::Main.any_instance.stubs(:container_available?).returns(true)
+
+    ensure_hook_runs("post-rollback")
+    run_command("rollback", "123")
+  end
+
   test "details" do
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:details")
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:details")
@@ -323,19 +335,6 @@ class CliMainTest < CliTestCase
     end
   end
 
-  test "broadcast" do
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with do |command, line, options, verbosity:|
-      command == "bin/audit_broadcast" &&
-        line =~ /\A'\[[^\]]+\] message'\z/ &&
-        options[:env].keys == %w[ MRSK_RECORDED_AT MRSK_PERFORMER MRSK_VERSION MRSK_EVENT ] &&
-        verbosity == :debug
-    end.returns("Broadcast audit message: message")
-
-    run_command("broadcast", "-m", "message").tap do |output|
-      assert_match "Broadcast: message", output
-    end
-  end
-
   test "version" do
     version = stdouted { Mrsk::Cli::Main.new.version }
     assert_equal Mrsk::VERSION, version

test/commands/auditor_test.rb

@@ -8,8 +8,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
     freeze_time
 
     @config = {
-      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ]
-      audit_broadcast_cmd: "bin/audit_broadcast"
     }
 
     @auditor = new_command
@@ -57,19 +56,6 @@ class CommandsAuditorTest < ActiveSupport::TestCase
     ], @auditor.record("app removed container", detail: "value")
   end
 
-  test "broadcast" do
-    assert_equal [
-      "bin/audit_broadcast",
-      "'[#{@performer}] [value] app removed container'",
-      env: {
-        "MRSK_RECORDED_AT" => @recorded_at,
-        "MRSK_PERFORMER" => @performer,
-        "MRSK_VERSION" => "123",
-        "MRSK_EVENT" => "app removed container",
-        "MRSK_DETAIL" => "value"
-      }
-    ], @auditor.broadcast("app removed container", detail: "value")
-  end
 
   private
     def new_command(destination: nil, **details)

test/fixtures/deploy_simple.yml

@@ -6,4 +6,3 @@ servers:
 registry:
   username: user
   password: pw
-audit_broadcast_cmd: "bin/audit_broadcast"

test/integration/docker/deployer/app/.mrsk/hooks/post-deploy

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "Deployed!"
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/post-deploy

test/integration/docker/deployer/app/.mrsk/hooks/post-rollback

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "Rolled back!"
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/post-rollback

test/integration/integration_test.rb

@@ -83,10 +83,10 @@ class IntegrationTest < ActiveSupport::TestCase
       assert_equal version, response.body.strip
     end
 
-    def assert_hooks_ran
+    def assert_hooks_ran(*hooks)
-      [ "pre-build" ].each do |hook|
+      hooks.each do |hook|
         file = "/tmp/#{ENV["TEST_ID"]}/#{hook}"
-        assert_match /File: #{file}/, deployer_exec("stat #{file}", capture: true)
+        assert_equal "removed '#{file}'", deployer_exec("rm -v #{file}", capture: true).strip
       end
     end
 

test/integration/main_test.rb

@@ -7,23 +7,20 @@ class MainTest < IntegrationTest
     assert_app_is_down
 
     mrsk :deploy
-
-    assert_hooks_ran
-
     assert_app_is_up version: first_version
+    assert_hooks_ran "pre-build", "post-deploy"
 
     second_version = update_app_rev
 
     mrsk :redeploy
-
     assert_app_is_up version: second_version
+    assert_hooks_ran "pre-build", "post-deploy"
 
     mrsk :rollback, first_version
-
+    assert_hooks_ran "post-rollback"
     assert_app_is_up version: first_version
 
     details = mrsk :details, capture: true
-
     assert_match /Traefik Host: vm1/, details
     assert_match /Traefik Host: vm2/, details
     assert_match /App Host: vm1/, details
@@ -32,7 +29,6 @@ class MainTest < IntegrationTest
     assert_match /registry:4443\/app:#{first_version}/, details
 
     audit = mrsk :audit, capture: true
-
     assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
   end