Merge pull request #291 from basecamp/hooks

MRSK Hooks

README.md

@@ -668,43 +668,6 @@ servers:
 
 This assumes the Cron settings are stored in `config/crontab`.
 
-### Using audit broadcasts
-
-If you'd like to broadcast audits of deploys, rollbacks, etc to a chatroom or elsewhere, you can configure the `audit_broadcast_cmd` setting with the path to a bin file that will be passed the audit line as the first argument:
-
-```yaml
-audit_broadcast_cmd:
-  bin/audit_broadcast
-```
-
-The broadcast command could look something like:
-
-```bash
-#!/usr/bin/env bash
-curl -q -d content="[My App] ${1}" https://3.basecamp.com/XXXXX/integrations/XXXXX/buckets/XXXXX/chats/XXXXX/lines
-```
-
-That'll post a line like follows to a preconfigured chatbot in Basecamp:
-
-```
-[My App] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de
-```
-
-`MRSK_*` environment variables are available to the broadcast command for
-fine-grained audit reporting, e.g. for triggering deployment reports or
-firing a JSON webhook. These variables include:
-- `MRSK_RECORDED_AT` - UTC timestamp in ISO 8601 format, e.g. `2023-04-14T17:07:31Z`
-- `MRSK_PERFORMER` - the local user performing the command (from `whoami`)
-- `MRSK_MESSAGE` - the full audit message, e.g. "Deployed app@150b24f"
-- `MRSK_DESTINATION` - optional: destination, e.g. "staging"
-- `MRSK_ROLE` - optional: role targeted, e.g. "web"
-
-Use `mrsk broadcast` to test and troubleshoot your broadcast command:
-
-```bash
-mrsk broadcast -m "test audit message"
-```
-
 ### Healthcheck
 
 MRSK uses Docker healtchecks to check the health of your application during deployment. Traefik uses this same healthcheck status to determine when a container is ready to receive traffic.
@@ -902,6 +865,52 @@ When `limit` is specified, containers will be booted on, at most, `limit` hosts
 
 These settings only apply when booting containers (using `mrsk deploy`, or `mrsk app boot`). For other commands, MRSK continues to run commands in parallel across all hosts.
 
+## Hooks
+
+You can run custom scripts at specific points with hooks.
+
+Hooks should be stored in the .mrsk/hooks folder. Running mrsk init will build that folder and add some sample scripts.
+
+You can change their location by setting `hooks_path` in the configuration file.
+
+If the script returns a non-zero exit code the command will be aborted.
+
+`MRSK_*` environment variables are available to the hooks command for
+fine-grained audit reporting, e.g. for triggering deployment reports or
+firing a JSON webhook. These variables include:
+- `MRSK_RECORDED_AT` - UTC timestamp in ISO 8601 format, e.g. `2023-04-14T17:07:31Z`
+- `MRSK_PERFORMER` - the local user performing the command (from `whoami`)
+- `MRSK_MESSAGE` - the full audit message, e.g. "Deployed app@150b24f"
+- `MRSK_SERVICE_VERSION` - an abbreviated version (for use in messages)
+- `MRSK_DESTINATION` - optional: destination, e.g. "staging"
+- `MRSK_ROLE` - optional: role targeted, e.g. "web"
+
+There are two hooks:
+
+1. pre-build
+Used for pre-build checks - e.g. there are no uncommitted changes or that CI has passed.
+
+2. post-deploy - run after a deploy, redeploy or rollback
+
+This hook is also passed a `MRSK_RUNTIME` env variable.
+
+This could be used to broadcast a deployment message, or register the new version with an APM.
+
+The command could look something like:
+
+```bash
+#!/usr/bin/env bash
+curl -q -d content="[My App] ${MRSK_PERFORMER} Rolled back to version ${MRSK_VERSION}" https://3.basecamp.com/XXXXX/integrations/XXXXX/buckets/XXXXX/chats/XXXXX/lines
+```
+
+That'll post a line like follows to a preconfigured chatbot in Basecamp:
+
+```
+[My App] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de
+```
+
+Set `--skip_hooks` to avoid running the hooks.
+
 ## Stage of development
 
 This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com).

lib/mrsk/cli.rb

@@ -1,5 +1,6 @@
 module Mrsk::Cli
   class LockError < StandardError; end
+  class HookError < StandardError; end
 end
 
 # SSHKit uses instance eval, so we need a global const for ergonomics

lib/mrsk/cli/accessory.rb

@@ -14,8 +14,6 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
             execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug
             execute *accessory.run
           end
-
-          audit_broadcast "Booted accessory #{name}" unless options[:skip_broadcast]
         end
       end
     end

lib/mrsk/cli/base.rb

@@ -20,7 +20,7 @@ module Mrsk::Cli
     class_option :config_file, aliases: "-c", default: "config/deploy.yml", desc: "Path to config file"
     class_option :destination, aliases: "-d", desc: "Specify destination to be used for config file (staging -> deploy.staging.yml)"
 
-    class_option :skip_broadcast, aliases: "-B", type: :boolean, default: false, desc: "Skip audit broadcasts"
+    class_option :skip_hooks, aliases: "-H", type: :boolean, default: false, desc: "Don't run hooks"
 
     def initialize(*)
       super
@@ -72,10 +72,6 @@ module Mrsk::Cli
         puts "  Finished all in #{sprintf("%.1f seconds", runtime)}"
       end
 
-      def audit_broadcast(line)
-        run_locally { execute *MRSK.auditor.broadcast(line), verbosity: :debug }
-      end
-
       def with_lock
         if MRSK.holding_lock?
           yield
@@ -134,5 +130,16 @@ module Mrsk::Cli
           MRSK.hold_lock_on_error = false
         end
       end
+
+      def run_hook(hook, **details)
+        if !options[:skip_hooks] && MRSK.hook.hook_exists?(hook)
+          say "Running the #{hook} hook...", :magenta
+          run_locally do
+            MRSK.with_verbosity(:debug) { execute *MRSK.hook.run(hook, **details) }
+          rescue SSHKit::Command::Failed
+            raise HookError.new("Hook `#{hook}` failed")
+          end
+        end
+      end
   end
 end

lib/mrsk/cli/build.rb

@@ -14,11 +14,12 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
     with_lock do
       cli = self
 
+      verify_local_dependencies
+      run_hook "pre-build"
+
       run_locally do
         begin
-          if cli.verify_local_dependencies
           MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
-          end
         rescue SSHKit::Command::Failed => e
           if e.message =~ /(no builder)|(no such file or directory)/
             error "Missing compatible builder, so creating a new one first"
@@ -82,8 +83,7 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
     end
   end
 
-
+  private
-  desc "", "" # Really a private method, but needed to be invoked from #push
     def verify_local_dependencies
       run_locally do
         begin
@@ -96,7 +96,5 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
           raise BuildError, build_error
         end
       end
-
-    true
     end
 end

lib/mrsk/cli/main.rb

@@ -44,7 +44,7 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       end
     end
 
-    audit_broadcast "Deployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
+    run_hook "post-deploy", runtime: runtime.round
   end
 
   desc "redeploy", "Deploy app to servers without bootstrapping servers, starting Traefik, pruning, and registry login"
@@ -72,11 +72,13 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       end
     end
 
-    audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
+    run_hook "post-deploy", runtime: runtime.round
   end
 
   desc "rollback [VERSION]", "Rollback app to VERSION"
   def rollback(version)
+    rolled_back = false
+    runtime = print_runtime do
       with_lock do
         invoke_options = deploy_options
 
@@ -85,14 +87,16 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
 
         if container_available?(version)
           invoke "mrsk:cli:app:boot", [], invoke_options.merge(version: version)
-
+          rolled_back = true
-        audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast]
         else
           say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
         end
       end
     end
 
+    run_hook "post-deploy", runtime: runtime.round if rolled_back
+  end
+
   desc "details", "Show details about all containers"
   def details
     invoke "mrsk:cli:traefik:details"
@@ -132,6 +136,14 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       puts "Created .env file"
     end
 
+    unless (hooks_dir = Pathname.new(File.expand_path(".mrsk/hooks"))).exist?
+      hooks_dir.mkpath
+      Pathname.new(File.expand_path("templates/sample_hooks", __dir__)).each_child do |sample_hook|
+        FileUtils.cp sample_hook, hooks_dir, preserve: true
+      end
+      puts "Created sample hooks in .mrsk/hooks"
+    end
+
     if options[:bundle]
       if (binstub = Pathname.new(File.expand_path("bin/mrsk"))).exist?
         puts "Binstub already exists in bin/mrsk (remove first to create a new one)"
@@ -172,13 +184,6 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
     end
   end
 
-  desc "broadcast", "Broadcast an audit message"
-  option :message, aliases: "-m", type: :string, desc: "Audit message", required: true
-  def broadcast
-    say "Broadcast: #{options[:message]}", :magenta
-    audit_broadcast options[:message]
-  end
-
   desc "version", "Show MRSK version"
   def version
     puts Mrsk::VERSION
@@ -235,8 +240,4 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
     def deploy_options
       { "version" => MRSK.config.version }.merge(options.without("skip_push"))
     end
-
-    def service_version(version = MRSK.config.abbreviated_version)
-      [ MRSK.config.service, version ].compact.join("@")
-    end
 end

lib/mrsk/cli/templates/deploy.yml

@@ -25,10 +25,6 @@ registry:
 #   secret:
 #     - RAILS_MASTER_KEY
 
-# Call a broadcast command on deploys.
-# audit_broadcast_cmd:
-#   bin/broadcast_to_bc
-
 # Use a different ssh user than root
 # ssh:
 #   user: app

lib/mrsk/cli/templates/sample_hooks/post-deploy.sample

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# A sample post-deploy hook
+#
+# Checks:
+# 1. We have a clean checkout
+# 2. A remote is configured
+# 3. The branch has been pushed to the remote
+# 4. The version we are deploying matches the remote
+#
+# These environment variables are available:
+# MRSK_RECORDED_AT
+# MRSK_PERFORMER
+# MRSK_VERSION
+# MRSK_DESTINATION (if set)
+# MRSK_RUNTIME
+
+echo "$MRSK_PERFORMER deployed $MRSK_VERSION to $MRSK_DESTINATION in $MRSK_RUNTIME seconds"

lib/mrsk/cli/templates/sample_hooks/pre-build.sample

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# A sample pre-build hook
+#
+# Checks:
+# 1. We have a clean checkout
+# 2. A remote is configured
+# 3. The branch has been pushed to the remote
+# 4. The version we are deploying matches the remote
+#
+# These environment variables are available:
+# MRSK_RECORDED_AT
+# MRSK_PERFORMER
+# MRSK_VERSION
+# MRSK_DESTINATION (if set)
+
+if [ -n "$(git status --porcelain)" ]; then
+  echo "Git checkout is not clean, aborting..." >&2
+  git status --porcelain >&2
+  exit 1
+fi
+
+first_remote=$(git remote)
+
+if [ -z "$first_remote" ]; then
+  echo "No git remote set, aborting..." >&2
+  exit 1
+fi
+
+current_branch=$(git branch --show-current)
+
+if [ -z "$current_branch" ]; then
+  echo "No git remote set, aborting..." >&2
+  exit 1
+fi
+
+remote_head=$(git ls-remote $first_remote --tags $current_branch | cut -f1)
+
+if [ -z "$remote_head" ]; then
+  echo "Branch not pushed to remote, aborting..." >&2
+  exit 1
+fi
+
+if [ "$MRSK_VERSION" != "$remote_head" ]; then
+  echo "Version ($MRSK_VERSION) does not match remote HEAD ($remote_head), aborting..." >&2
+  exit 1
+fi
+
+exit 0

lib/mrsk/commander.rb

@@ -100,6 +100,14 @@ class Mrsk::Commander
     @healthcheck ||= Mrsk::Commands::Healthcheck.new(config)
   end
 
+  def hook
+    @hook ||= Mrsk::Commands::Hook.new(config)
+  end
+
+  def lock
+    @lock ||= Mrsk::Commands::Lock.new(config)
+  end
+
   def prune
     @prune ||= Mrsk::Commands::Prune.new(config)
   end
@@ -112,10 +120,6 @@ class Mrsk::Commander
     @traefik ||= Mrsk::Commands::Traefik.new(config)
   end
 
-  def lock
-    @lock ||= Mrsk::Commands::Lock.new(config)
-  end
-
   def with_verbosity(level)
     old_level = self.verbosity
 

lib/mrsk/commands/auditor.rb

@@ -1,27 +1,18 @@
-require "time"
-
 class Mrsk::Commands::Auditor < Mrsk::Commands::Base
   attr_reader :details
 
   def initialize(config, **details)
     super(config)
-    @details = default_details.merge(details)
+    @details = details
   end
 
   # Runs remotely
   def record(line, **details)
     append \
-      [ :echo, *audit_tags(**details), line ],
+      [ :echo, audit_tags(**details).except(:version, :service_version).to_s, line ],
       audit_log_file
   end
 
-  # Runs locally
-  def broadcast(line, **details)
-    if broadcast_cmd = config.audit_broadcast_cmd
-      [ broadcast_cmd, *broadcast_args(line, **details), env: env_for(event: line, **details) ]
-    end
-  end
-
   def reveal
     [ :tail, "-n", 50, audit_log_file ]
   end
@@ -31,29 +22,7 @@ class Mrsk::Commands::Auditor < Mrsk::Commands::Base
       [ "mrsk", config.service, config.destination, "audit.log" ].compact.join("-")
     end
 
-    def default_details
-      { recorded_at: Time.now.utc.iso8601,
-        performer: `whoami`.chomp,
-        destination: config.destination }
-    end
-
     def audit_tags(**details)
-      tags_for **self.details.merge(details)
+      tags(**self.details, **details)
-    end
-
-    def broadcast_args(line, **details)
-      "'#{broadcast_tags(**details).join(" ")} #{line}'"
-    end
-
-    def broadcast_tags(**details)
-      tags_for **self.details.merge(details).except(:recorded_at)
-    end
-
-    def tags_for(**details)
-      details.compact.values.map { |value| "[#{value}]" }
-    end
-
-    def env_for(**details)
-      self.details.merge(details).compact.transform_keys { |detail| "MRSK_#{detail.upcase}" }
     end
 end

lib/mrsk/commands/base.rb

@@ -53,5 +53,9 @@ module Mrsk::Commands
       def docker(*args)
         args.compact.unshift :docker
       end
+
+      def tags(**details)
+        Mrsk::Tags.from_config(config, **details)
+      end
   end
 end

lib/mrsk/commands/hook.rb

@@ -0,0 +1,14 @@
+class Mrsk::Commands::Hook < Mrsk::Commands::Base
+  def run(hook, **details)
+    [ hook_file(hook), env: tags(**details).env ]
+  end
+
+  def hook_exists?(hook)
+    Pathname.new(hook_file(hook)).exist?
+  end
+
+  private
+    def hook_file(hook)
+      "#{config.hooks_path}/#{hook}"
+    end
+end

lib/mrsk/configuration.rb

@@ -6,7 +6,7 @@ require "erb"
 require "net/ssh/proxy/jump"
 
 class Mrsk::Configuration
-  delegate :service, :image, :servers, :env, :labels, :registry, :builder, :stop_wait_time, to: :raw_config, allow_nil: true
+  delegate :service, :image, :servers, :env, :labels, :registry, :builder, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
   delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Mrsk::Utils
 
   attr_accessor :destination
@@ -157,10 +157,6 @@ class Mrsk::Configuration
   end
 
 
-  def audit_broadcast_cmd
-    raw_config.audit_broadcast_cmd
-  end
-
   def healthcheck
     { "path" => "/up", "port" => 3000, "max_attempts" => 7 }.merge(raw_config.healthcheck || {})
   end
@@ -197,6 +193,10 @@ class Mrsk::Configuration
     raw_config.traefik || {}
   end
 
+  def hooks_path
+    raw_config.hooks_path || ".mrsk/hooks"
+  end
+
   private
     # Will raise ArgumentError if any required config keys are missing
     def ensure_required_keys_present

lib/mrsk/tags.rb

@@ -0,0 +1,39 @@
+require "time"
+
+class Mrsk::Tags
+  attr_reader :config, :tags
+
+  class << self
+    def from_config(config, **extra)
+      new(**default_tags(config), **extra)
+    end
+
+    def default_tags(config)
+      { recorded_at: Time.now.utc.iso8601,
+        performer: `whoami`.chomp,
+        destination: config.destination,
+        version: config.version,
+        service_version: service_version(config) }
+    end
+
+    def service_version(config)
+      [ config.service, config.abbreviated_version ].compact.join("@")
+    end
+  end
+
+  def initialize(**tags)
+    @tags = tags.compact
+  end
+
+  def env
+    tags.transform_keys { |detail| "MRSK_#{detail.upcase}" }
+  end
+
+  def to_s
+    tags.values.map { |value| "[#{value}]" }.join(" ")
+  end
+
+  def except(*tags)
+    self.class.new(**self.tags.except(*tags))
+  end
+end

lib/mrsk/utils.rb

@@ -84,6 +84,13 @@ module Mrsk::Utils
 
   # Abbreviate a git revhash for concise display
   def abbreviate_version(version)
-    version[0...7] if version
+    if version
+      # Don't abbreviate <sha>_uncommitted_<etc>
+      if version.include?("_")
+        version
+      else
+        version[0...7]
+      end
+    end
   end
 end

test/cli/app_test.rb

@@ -2,12 +2,7 @@ require_relative "cli_test_case"
 
 class CliAppTest < CliTestCase
   test "boot" do
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
+    stub_running
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-    .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-    .returns("running") # health check
-
     run_command("boot").tap do |output|
       assert_match "docker tag dhh/app:latest dhh/app:latest", output
       assert_match "docker run --detach --restart unless-stopped", output
@@ -51,7 +46,7 @@ class CliAppTest < CliTestCase
   end
 
   test "boot errors leave lock in place" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
 
     Mrsk::Cli::App.any_instance.expects(:using_version).raises(RuntimeError)
 
@@ -183,4 +178,12 @@ class CliAppTest < CliTestCase
     def run_command(*command, config: :with_accessories)
       stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1"]) }
     end
+
+    def stub_running
+      SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+        .returns("running") # health check
+    end
 end

test/cli/build_test.rb

@@ -9,17 +9,19 @@ class CliBuildTest < CliTestCase
   end
 
   test "push" do
-    Mrsk::Cli::Build.any_instance.stubs(:verify_local_dependencies).returns(true)
     run_command("push").tap do |output|
+      assert_match /docker --version && docker buildx version/, output
       assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder mrsk-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
     end
   end
 
   test "push without builder" do
     stub_locking
-    Mrsk::Cli::Build.any_instance.stubs(:verify_local_dependencies).returns(true)
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |arg| arg == :docker }
+      .with(:docker, "--version", "&&", :docker, :buildx, "version")
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |*args| args[0..1] == [:docker, :buildx] }
       .raises(SSHKit::Command::Failed.new("no builder"))
       .then
       .returns(true)
@@ -29,6 +31,24 @@ class CliBuildTest < CliTestCase
     end
   end
 
+  test "push with no buildx plugin" do
+    stub_locking
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with(:docker, "--version", "&&", :docker, :buildx, "version")
+      .raises(SSHKit::Command::Failed.new("no buildx"))
+
+    Mrsk::Commands::Builder.any_instance.stubs(:native_and_local?).returns(false)
+    assert_raises(Mrsk::Cli::Build::BuildError) { run_command("push") }
+  end
+
+  test "push pre-build hook failure" do
+    fail_hook("pre-build")
+
+    assert_raises(Mrsk::Cli::HookError) { run_command("push") }
+
+    assert @executions.none? { |args| args[0..2] == [:docker, :buildx, :build] }
+  end
+
   test "pull" do
     run_command("pull").tap do |output|
       assert_match /docker image rm --force dhh\/app:999/, output
@@ -70,32 +90,15 @@ class CliBuildTest < CliTestCase
     end
   end
 
-  test "verify local dependencies" do
-    Mrsk::Commands::Builder.any_instance.stubs(:name).returns("remote".inquiry)
-
-    run_command("verify_local_dependencies").tap do |output|
-      assert_match /docker --version && docker buildx version/, output
-    end
-  end
-
-  test "verify local dependencies with no buildx plugin" do
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, "--version", "&&", :docker, :buildx, "version")
-      .raises(SSHKit::Command::Failed.new("no buildx"))
-
-    Mrsk::Commands::Builder.any_instance.stubs(:native_and_local?).returns(false)
-    assert_raises(Mrsk::Cli::Build::BuildError) { run_command("verify_local_dependencies") }
-  end
-
   private
     def run_command(*command)
       stdouted { Mrsk::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
     end
 
-    def stub_locking
+    def stub_dependency_checks
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock }
+        .with(:docker, "--version", "&&", :docker, :buildx, "version")
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" }
+        .with { |*args| args[0..1] == [:docker, :buildx] }
     end
 end

test/cli/cli_test_case.rb

@@ -14,4 +14,32 @@ class CliTestCase < ActiveSupport::TestCase
     ENV.delete("MYSQL_ROOT_PASSWORD")
     ENV.delete("VERSION")
   end
+
+  private
+    def fail_hook(hook)
+      @executions = []
+      Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| @executions << args; args != [".mrsk/hooks/#{hook}"] }
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| args.first == ".mrsk/hooks/#{hook}" }
+        .raises(SSHKit::Command::Failed.new("failed"))
+    end
+
+    def ensure_hook_runs(hook)
+      Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| args != [".mrsk/hooks/#{hook}"] }
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with { |*args| args.first == ".mrsk/hooks/#{hook}" }
+        .once
+    end
+
+    def stub_locking
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock }
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" }
+    end
 end

test/cli/main_test.rb

@@ -10,7 +10,7 @@ class CliMainTest < CliTestCase
   end
 
   test "deploy" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
@@ -20,6 +20,8 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
 
+    Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+
     run_command("deploy").tap do |output|
       assert_match /Log into image registry/, output
       assert_match /Build and push app image/, output
@@ -27,11 +29,12 @@ class CliMainTest < CliTestCase
       assert_match /Ensure app can pass healthcheck/, output
       assert_match /Detect stale containers/, output
       assert_match /Prune old containers and images/, output
+      assert_match /Running the post-deploy hook.../, output
     end
   end
 
   test "deploy with skip_push" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
@@ -81,7 +84,7 @@ class CliMainTest < CliTestCase
   end
 
   test "deploy errors during outside section leave remove lock" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke)
       .with("mrsk:cli:registry:login", [], invoke_options)
@@ -94,22 +97,41 @@ class CliMainTest < CliTestCase
     assert !MRSK.holding_lock?
   end
 
+  test "deploy with skipped hooks" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => true }
+
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
+
+    run_command("deploy", "--skip_hooks") do
+      refute_match /Running the post-deploy hook.../, output
+    end
+  end
+
   test "redeploy" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
 
+    Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+
     run_command("redeploy").tap do |output|
       assert_match /Build and push app image/, output
       assert_match /Ensure app can pass healthcheck/, output
+      assert_match /Running the post-deploy hook.../, output
     end
   end
 
   test "redeploy with skip_push" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
@@ -149,12 +171,14 @@ class CliMainTest < CliTestCase
         .returns("running").at_least_once # health check
     end
 
+    Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
 
     run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
       assert_match "Start container with version 123", output
       assert_match "docker tag dhh/app:123 dhh/app:latest", output
       assert_match "docker start app-web-123", output
       assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running"
+      assert_match "Running the post-deploy hook...", output
     end
   end
 
@@ -235,9 +259,11 @@ class CliMainTest < CliTestCase
   end
 
   test "init" do
-    Pathname.any_instance.expects(:exist?).returns(false).twice
+    Pathname.any_instance.expects(:exist?).returns(false).times(3)
+    Pathname.any_instance.stubs(:mkpath)
     FileUtils.stubs(:mkdir_p)
     FileUtils.stubs(:cp_r)
+    FileUtils.stubs(:cp)
 
     run_command("init").tap do |output|
       assert_match /Created configuration file in config\/deploy.yml/, output
@@ -246,7 +272,7 @@ class CliMainTest < CliTestCase
   end
 
   test "init with existing config" do
-    Pathname.any_instance.expects(:exist?).returns(true).twice
+    Pathname.any_instance.expects(:exist?).returns(true).times(3)
 
     run_command("init").tap do |output|
       assert_match /Config file already exists in config\/deploy.yml \(remove first to create a new one\)/, output
@@ -254,9 +280,11 @@ class CliMainTest < CliTestCase
   end
 
   test "init with bundle option" do
-    Pathname.any_instance.expects(:exist?).returns(false).times(3)
+    Pathname.any_instance.expects(:exist?).returns(false).times(4)
+    Pathname.any_instance.stubs(:mkpath)
     FileUtils.stubs(:mkdir_p)
     FileUtils.stubs(:cp_r)
+    FileUtils.stubs(:cp)
 
     run_command("init", "--bundle").tap do |output|
       assert_match /Created configuration file in config\/deploy.yml/, output
@@ -269,9 +297,11 @@ class CliMainTest < CliTestCase
   end
 
   test "init with bundle option and existing binstub" do
-    Pathname.any_instance.expects(:exist?).returns(true).times(3)
+    Pathname.any_instance.expects(:exist?).returns(true).times(4)
+    Pathname.any_instance.stubs(:mkpath)
     FileUtils.stubs(:mkdir_p)
     FileUtils.stubs(:cp_r)
+    FileUtils.stubs(:cp)
 
     run_command("init", "--bundle").tap do |output|
       assert_match /Config file already exists in config\/deploy.yml \(remove first to create a new one\)/, output
@@ -317,19 +347,6 @@ class CliMainTest < CliTestCase
     end
   end
 
-  test "broadcast" do
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with do |command, line, options, verbosity:|
-      command == "bin/audit_broadcast" &&
-        line =~ /\A'\[[^\]]+\] message'\z/ &&
-        options[:env].keys == %w[ MRSK_RECORDED_AT MRSK_PERFORMER MRSK_EVENT ] &&
-        verbosity == :debug
-    end.returns("Broadcast audit message: message")
-
-    run_command("broadcast", "-m", "message").tap do |output|
-      assert_match "Broadcast: message", output
-    end
-  end
-
   test "version" do
     version = stdouted { Mrsk::Cli::Main.new.version }
     assert_equal Mrsk::VERSION, version

test/commands/auditor_test.rb

@@ -1,19 +1,25 @@
 require "test_helper"
+require "active_support/testing/time_helpers"
 
 class CommandsAuditorTest < ActiveSupport::TestCase
+  include ActiveSupport::Testing::TimeHelpers
+
   setup do
+    freeze_time
+
     @config = {
-      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ]
-      audit_broadcast_cmd: "bin/audit_broadcast"
     }
 
     @auditor = new_command
+    @performer = `whoami`.strip
+    @recorded_at = Time.now.utc.iso8601
   end
 
   test "record" do
     assert_equal [
       :echo,
-      "[#{@auditor.details[:recorded_at]}]", "[#{@auditor.details[:performer]}]",
+      "[#{@recorded_at}] [#{@performer}]",
       "app removed container",
       ">>", "mrsk-app-audit.log"
     ], @auditor.record("app removed container")
@@ -23,7 +29,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
     new_command(destination: "staging").tap do |auditor|
       assert_equal [
         :echo,
-        "[#{auditor.details[:recorded_at]}]", "[#{auditor.details[:performer]}]", "[#{auditor.details[:destination]}]",
+        "[#{@recorded_at}] [#{@performer}] [staging]",
         "app removed container",
         ">>", "mrsk-app-staging-audit.log"
       ], auditor.record("app removed container")
@@ -34,7 +40,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
     new_command(role: "web").tap do |auditor|
       assert_equal [
         :echo,
-        "[#{auditor.details[:recorded_at]}]", "[#{auditor.details[:performer]}]", "[#{auditor.details[:role]}]",
+        "[#{@recorded_at}] [#{@performer}] [web]",
         "app removed container",
         ">>", "mrsk-app-audit.log"
       ], auditor.record("app removed container")
@@ -44,24 +50,12 @@ class CommandsAuditorTest < ActiveSupport::TestCase
   test "record with arg details" do
     assert_equal [
       :echo,
-      "[#{@auditor.details[:recorded_at]}]", "[#{@auditor.details[:performer]}]", "[value]",
+      "[#{@recorded_at}] [#{@performer}] [value]",
       "app removed container",
       ">>", "mrsk-app-audit.log"
     ], @auditor.record("app removed container", detail: "value")
   end
 
-  test "broadcast" do
-    assert_equal [
-      "bin/audit_broadcast",
-      "'[#{@auditor.details[:performer]}] [value] app removed container'",
-      env: {
-        "MRSK_RECORDED_AT" => @auditor.details[:recorded_at],
-        "MRSK_PERFORMER" => @auditor.details[:performer],
-        "MRSK_EVENT" => "app removed container",
-        "MRSK_DETAIL" => "value"
-      }
-    ], @auditor.broadcast("app removed container", detail: "value")
-  end
 
   private
     def new_command(destination: nil, **details)

test/commands/hook_test.rb

@@ -0,0 +1,44 @@
+require "test_helper"
+
+class CommandsHookTest < ActiveSupport::TestCase
+  include ActiveSupport::Testing::TimeHelpers
+
+  setup do
+    freeze_time
+
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
+    }
+
+    @performer = `whoami`.strip
+    @recorded_at = Time.now.utc.iso8601
+  end
+
+  test "run" do
+    assert_equal [
+      ".mrsk/hooks/foo",
+      { env: {
+        "MRSK_RECORDED_AT" => @recorded_at,
+        "MRSK_PERFORMER" => @performer,
+        "MRSK_VERSION" => "123",
+        "MRSK_SERVICE_VERSION" => "app@123" } }
+    ], new_command.run("foo")
+  end
+
+  test "run with custom hooks_path" do
+    assert_equal [
+      "custom/hooks/path/foo",
+      { env: {
+        "MRSK_RECORDED_AT" => @recorded_at,
+        "MRSK_PERFORMER" => @performer,
+        "MRSK_VERSION" => "123",
+        "MRSK_SERVICE_VERSION" => "app@123" } }
+    ], new_command(hooks_path: "custom/hooks/path").run("foo")
+  end
+
+  private
+    def new_command(**extra_config)
+      Mrsk::Commands::Hook.new(Mrsk::Configuration.new(@config.merge(**extra_config), version: "123"))
+    end
+end

test/fixtures/deploy_simple.yml

@@ -6,4 +6,3 @@ servers:
 registry:
   username: user
   password: pw
-audit_broadcast_cmd: "bin/audit_broadcast"

test/integration/docker-compose.yml

@@ -17,6 +17,8 @@ services:
     privileged: true
     build:
       context: docker/deployer
+    environment:
+      - TEST_ID=${TEST_ID}
     volumes:
       - ../..:/mrsk
       - shared:/shared

test/integration/docker/deployer/app/.mrsk/hooks/post-deploy

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "Deployed!"
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/post-deploy

test/integration/docker/deployer/app/.mrsk/hooks/pre-build

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "About to build and push..."
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-build

test/integration/docker/deployer/boot.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-dockerd &
+dockerd --max-concurrent-downloads 1 &
 
 exec sleep infinity

test/integration/docker/vm/boot.sh

@@ -4,6 +4,6 @@ while [ ! -f /root/.ssh/authorized_keys ]; do echo "Waiting for ssh keys"; sleep
 
 service ssh restart
 
-dockerd &
+dockerd --max-concurrent-downloads 1 &
 
 exec sleep infinity

test/integration/integration_test.rb

@@ -3,6 +3,7 @@ require "test_helper"
 
 class IntegrationTest < ActiveSupport::TestCase
   setup do
+    ENV["TEST_ID"] = SecureRandom.hex
     docker_compose "up --build -d"
     wait_for_healthy
     setup_deployer
@@ -14,7 +15,7 @@ class IntegrationTest < ActiveSupport::TestCase
 
   private
     def docker_compose(*commands, capture: false, raise_on_error: true)
-      command = "docker compose #{commands.join(" ")}"
+      command = "TEST_ID=#{ENV["TEST_ID"]} docker compose #{commands.join(" ")}"
       succeeded = false
       if capture
         result = stdouted { succeeded = system("cd test/integration && #{command}") }
@@ -82,6 +83,25 @@ class IntegrationTest < ActiveSupport::TestCase
       assert_equal version, response.body.strip
     end
 
+    def assert_hooks_ran(*hooks)
+      hooks.each do |hook|
+        file = "/tmp/#{ENV["TEST_ID"]}/#{hook}"
+        assert_equal "removed '#{file}'", deployer_exec("rm -v #{file}", capture: true).strip
+      end
+    end
+
+    def assert_200(response)
+      code = response.code
+      if code != "200"
+        puts "Got response code #{code}, here are the traefik logs:"
+        mrsk :traefik, :logs
+        puts "And here are the load balancer logs"
+        docker_compose :logs, :load_balancer
+        puts "Tried to get the response code again and got #{app_response.code}"
+      end
+      assert_equal "200", code
+    end
+
     def wait_for_healthy(timeout: 20)
       timeout_at = Time.now + timeout
       while docker_compose("ps -a | tail -n +2 | grep -v '(healthy)' | wc -l", capture: true) != "0"

test/integration/main_test.rb

@@ -7,21 +7,20 @@ class MainTest < IntegrationTest
     assert_app_is_down
 
     mrsk :deploy
-
     assert_app_is_up version: first_version
+    assert_hooks_ran "pre-build", "post-deploy"
 
     second_version = update_app_rev
 
     mrsk :redeploy
-
     assert_app_is_up version: second_version
+    assert_hooks_ran "pre-build", "post-deploy"
 
     mrsk :rollback, first_version
-
+    assert_hooks_ran "post-deploy"
     assert_app_is_up version: first_version
 
     details = mrsk :details, capture: true
-
     assert_match /Traefik Host: vm1/, details
     assert_match /Traefik Host: vm2/, details
     assert_match /App Host: vm1/, details
@@ -30,7 +29,6 @@ class MainTest < IntegrationTest
     assert_match /registry:4443\/app:#{first_version}/, details
 
     audit = mrsk :audit, capture: true
-
     assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
   end