MRSK hooks

Adds hooks to MRSK. Currently just two hooks, pre-build and post-push.

We could break the build and push into two separate commands if we
found the need for post-build and/or pre-push hooks.

Hooks are stored in `.mrsk/hooks`. Running `mrsk init` will now create
that folder and add sample hook scripts.

Hooks returning non-zero exit codes will abort the current command.

Further potential work here:
- We could replace the audit broadcast command with a
post-deploy/post-rollback hook or similar
- Maybe provide pre-command/post-command hooks that run after every
mrsk invocation
- Also look for hooks in `~/.mrsk/hooks`

README.md

@@ -902,6 +902,19 @@ When `limit` is specified, containers will be booted on, at most, `limit` hosts
 
 These settings only apply when booting containers (using `mrsk deploy`, or `mrsk app boot`). For other commands, MRSK continues to run commands in parallel across all hosts.
 
+## Hooks
+
+You can run custom scripts at specific points with hooks.
+
+Hooks should be stored in the .mrsk/hooks folder. Running mrsk init will build that folder and add some sample scripts.
+
+If the script returns a non-zero exit code the command will be aborted.
+
+There are currently two hooks:
+
+- pre-build
+- post-push
+
 ## Stage of development
 
 This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com).

lib/mrsk/cli.rb

@@ -1,5 +1,6 @@
 module Mrsk::Cli
   class LockError < StandardError; end
+  class HookError < StandardError; end
 end
 
 # SSHKit uses instance eval, so we need a global const for ergonomics

lib/mrsk/cli/base.rb

@@ -134,5 +134,17 @@ module Mrsk::Cli
           MRSK.hold_lock_on_error = false
         end
       end
+
+      def run_hook(hook)
+        run_locally do
+          if MRSK.hook.hook_exists?(hook)
+            begin
+              MRSK.with_verbosity(:debug) { execute(*MRSK.hook.run(hook)) }
+            rescue SSHKit::Command::Failed
+              raise HookError.new("Hook `#{hook}` failed")
+            end
+          end
+        end
+      end
   end
 end

lib/mrsk/cli/build.rb

@@ -16,6 +16,7 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
 
       verify_local_dependencies
 
+      run_hook("pre-build")
       run_locally do
         begin
           MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
@@ -31,6 +32,7 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
           end
         end
       end
+      run_hook("post-push")
     end
   end
 

lib/mrsk/cli/main.rb

@@ -132,6 +132,14 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       puts "Created .env file"
     end
 
+    unless (hooks_dir = Pathname.new(File.expand_path(".mrsk/hooks"))).exist?
+      hooks_dir.mkpath
+      Pathname.new(File.expand_path("templates/sample_hooks", __dir__)).each_child do |sample_hook|
+        FileUtils.cp sample_hook, hooks_dir
+      end
+      puts "Created sample hooks in .mrsk/hooks"
+    end
+
     if options[:bundle]
       if (binstub = Pathname.new(File.expand_path("bin/mrsk"))).exist?
         puts "Binstub already exists in bin/mrsk (remove first to create a new one)"

lib/mrsk/cli/templates/sample_hooks/post-push.sample

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# A sample post-push hook
+#
+# Checks:
+# 1. We have a clean checkout
+#
+# These environment variables are available:
+# MRSK_RECORDED_AT
+# MRSK_PERFORMER
+# MRSK_VERSION
+# MRSK_DESTINATION (if set)
+
+if [ -n "$(git status --porcelain)" ]; then
+  echo "Git checkout is not clean, aborting..."
+  git status --porcelain >&2
+  exit 1
+fi
+
+exit 0

lib/mrsk/cli/templates/sample_hooks/pre-build.sample

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# A sample pre-build hook
+#
+# Checks:
+# 1. We have a clean checkout
+# 2. A remote is configured
+# 3. The branch has been pushed to the remote
+# 4. The version we are deploying matches the remote
+#
+# These environment variables are available:
+# MRSK_RECORDED_AT
+# MRSK_PERFORMER
+# MRSK_VERSION
+# MRSK_DESTINATION (if set)
+
+if [ -n "$(git status --porcelain)" ]; then
+  echo "Git checkout is not clean, aborting..." >&2
+  git status --porcelain >&2
+  exit 1
+fi
+
+first_remote=$(git remote)
+
+if [ -z "$first_remote" ]; then
+  echo "No git remote set, aborting..." >&2
+  exit 1
+fi
+
+current_branch=$(git branch --show-current)
+
+if [ -z "$current_branch" ]; then
+  echo "No git remote set, aborting..." >&2
+  exit 1
+fi
+
+remote_head=$(git ls-remote $first_remote --tags $current_branch | cut -f1)
+
+if [ -z "$remote_head" ]; then
+  echo "Branch not pushed to remote, aborting..." >&2
+  exit 1
+fi
+
+if [ "$MRSK_VERSION" != "$remote_head" ]; then
+  echo "Version ($MRSK_VERSION) does not match remote HEAD ($remote_head), aborting..." >&2
+  exit 1
+fi
+
+exit 0

lib/mrsk/commander.rb

@@ -100,6 +100,14 @@ class Mrsk::Commander
     @healthcheck ||= Mrsk::Commands::Healthcheck.new(config)
   end
 
+  def hook
+    @hook ||= Mrsk::Commands::Hook.new(config)
+  end
+
+  def lock
+    @lock ||= Mrsk::Commands::Lock.new(config)
+  end
+
   def prune
     @prune ||= Mrsk::Commands::Prune.new(config)
   end
@@ -112,10 +120,6 @@ class Mrsk::Commander
     @traefik ||= Mrsk::Commands::Traefik.new(config)
   end
 
-  def lock
-    @lock ||= Mrsk::Commands::Lock.new(config)
-  end
-
   def with_verbosity(level)
     old_level = self.verbosity
 

lib/mrsk/commands/auditor.rb

@@ -1,24 +1,23 @@
-require "time"
-
 class Mrsk::Commands::Auditor < Mrsk::Commands::Base
   attr_reader :details
 
   def initialize(config, **details)
     super(config)
-    @details = default_details.merge(details)
+    @details = details
   end
 
   # Runs remotely
   def record(line, **details)
     append \
-      [ :echo, *audit_tags(**details), line ],
+      [ :echo, audit_tags(**details).except(:version).to_s, line ],
       audit_log_file
   end
 
   # Runs locally
   def broadcast(line, **details)
     if broadcast_cmd = config.audit_broadcast_cmd
-      [ broadcast_cmd, *broadcast_args(line, **details), env: env_for(event: line, **details) ]
+      tags = audit_tags(**details, event: line)
+      [ broadcast_cmd,  "'#{tags.except(:recorded_at, :event, :version)} #{line}'", env: tags.env ]
     end
   end
 
@@ -31,29 +30,7 @@ class Mrsk::Commands::Auditor < Mrsk::Commands::Base
       [ "mrsk", config.service, config.destination, "audit.log" ].compact.join("-")
     end
 
-    def default_details
-      { recorded_at: Time.now.utc.iso8601,
-        performer: `whoami`.chomp,
-        destination: config.destination }
-    end
-
     def audit_tags(**details)
-      tags_for **self.details.merge(details)
+      tags(**self.details, **details)
-    end
-
-    def broadcast_args(line, **details)
-      "'#{broadcast_tags(**details).join(" ")} #{line}'"
-    end
-
-    def broadcast_tags(**details)
-      tags_for **self.details.merge(details).except(:recorded_at)
-    end
-
-    def tags_for(**details)
-      details.compact.values.map { |value| "[#{value}]" }
-    end
-
-    def env_for(**details)
-      self.details.merge(details).compact.transform_keys { |detail| "MRSK_#{detail.upcase}" }
     end
 end

lib/mrsk/commands/base.rb

@@ -53,5 +53,9 @@ module Mrsk::Commands
       def docker(*args)
         args.compact.unshift :docker
       end
+
+      def tags(**details)
+        Mrsk::Tags.from_config(config, **details)
+      end
   end
 end

lib/mrsk/commands/hook.rb

@@ -0,0 +1,14 @@
+class Mrsk::Commands::Hook < Mrsk::Commands::Base
+  def run(hook, **details)
+    [ ".mrsk/hooks/#{hook}", env: tags(**details).env ]
+  end
+
+  def hook_exists?(hook)
+    Pathname.new(hook_file(hook)).exist?
+  end
+
+  private
+    def hook_file(hook)
+      ".mrsk/hooks/#{hook}"
+    end
+end

lib/mrsk/tags.rb

@@ -0,0 +1,34 @@
+require "time"
+
+class Mrsk::Tags
+  attr_reader :config, :tags
+
+  class << self
+    def from_config(config, **extra)
+      new(**default_tags(config), **extra)
+    end
+
+    def default_tags(config)
+      { recorded_at: Time.now.utc.iso8601,
+        performer: `whoami`.chomp,
+        destination: config.destination,
+        version: config.version }
+    end
+  end
+
+  def initialize(**tags)
+    @tags = tags.compact
+  end
+
+  def env
+    tags.transform_keys { |detail| "MRSK_#{detail.upcase}" }
+  end
+
+  def to_s
+    tags.values.map { |value| "[#{value}]" }.join(" ")
+  end
+
+  def except(*tags)
+    self.class.new(**self.tags.except(*tags))
+  end
+end

test/cli/app_test.rb

@@ -2,12 +2,7 @@ require_relative "cli_test_case"
 
 class CliAppTest < CliTestCase
   test "boot" do
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
+    stub_running
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-    .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-    .returns("running") # health check
-
     run_command("boot").tap do |output|
       assert_match "docker tag dhh/app:latest dhh/app:latest", output
       assert_match "docker run --detach --restart unless-stopped", output
@@ -183,4 +178,12 @@ class CliAppTest < CliTestCase
     def run_command(*command, config: :with_accessories)
       stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1"]) }
     end
+
+    def stub_running
+      SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+        .returns("running") # health check
+    end
 end

test/cli/build_test.rb

@@ -41,6 +41,22 @@ class CliBuildTest < CliTestCase
     assert_raises(Mrsk::Cli::Build::BuildError) { run_command("push") }
   end
 
+  test "push pre-build hook failure" do
+    fail_hook("pre-build")
+
+    assert_raises(Mrsk::Cli::HookError) { run_command("push") }
+
+    assert @executions.none? { |args| args[0..2] == [:docker, :buildx, :build] }
+  end
+
+  test "push post-push hook failure" do
+    fail_hook("post-push")
+
+    assert_raises(Mrsk::Cli::HookError) { run_command("push") }
+
+    assert @executions.any? { |args| args[0..2] == [:docker, :buildx, :build] }
+  end
+
   test "pull" do
     run_command("pull").tap do |output|
       assert_match /docker image rm --force dhh\/app:999/, output
@@ -93,4 +109,11 @@ class CliBuildTest < CliTestCase
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
         .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" }
     end
+
+    def stub_dependency_checks
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with(:docker, "--version", "&&", :docker, :buildx, "version")
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| args[0..1] == [:docker, :buildx] }
+    end
 end

test/cli/cli_test_case.rb

@@ -14,4 +14,17 @@ class CliTestCase < ActiveSupport::TestCase
     ENV.delete("MYSQL_ROOT_PASSWORD")
     ENV.delete("VERSION")
   end
+
+  private
+    def fail_hook(hook)
+      @executions = []
+      Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| @executions << args; args != [".mrsk/hooks/#{hook}"] }
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| args.first == ".mrsk/hooks/#{hook}" }
+        .raises(SSHKit::Command::Failed.new("failed"))
+    end
+
 end

test/cli/main_test.rb

@@ -235,9 +235,11 @@ class CliMainTest < CliTestCase
   end
 
   test "init" do
-    Pathname.any_instance.expects(:exist?).returns(false).twice
+    Pathname.any_instance.expects(:exist?).returns(false).times(3)
+    Pathname.any_instance.stubs(:mkpath)
     FileUtils.stubs(:mkdir_p)
     FileUtils.stubs(:cp_r)
+    FileUtils.stubs(:cp)
 
     run_command("init").tap do |output|
       assert_match /Created configuration file in config\/deploy.yml/, output
@@ -246,7 +248,7 @@ class CliMainTest < CliTestCase
   end
 
   test "init with existing config" do
-    Pathname.any_instance.expects(:exist?).returns(true).twice
+    Pathname.any_instance.expects(:exist?).returns(true).times(3)
 
     run_command("init").tap do |output|
       assert_match /Config file already exists in config\/deploy.yml \(remove first to create a new one\)/, output
@@ -254,9 +256,11 @@ class CliMainTest < CliTestCase
   end
 
   test "init with bundle option" do
-    Pathname.any_instance.expects(:exist?).returns(false).times(3)
+    Pathname.any_instance.expects(:exist?).returns(false).times(4)
+    Pathname.any_instance.stubs(:mkpath)
     FileUtils.stubs(:mkdir_p)
     FileUtils.stubs(:cp_r)
+    FileUtils.stubs(:cp)
 
     run_command("init", "--bundle").tap do |output|
       assert_match /Created configuration file in config\/deploy.yml/, output
@@ -269,9 +273,11 @@ class CliMainTest < CliTestCase
   end
 
   test "init with bundle option and existing binstub" do
-    Pathname.any_instance.expects(:exist?).returns(true).times(3)
+    Pathname.any_instance.expects(:exist?).returns(true).times(4)
+    Pathname.any_instance.stubs(:mkpath)
     FileUtils.stubs(:mkdir_p)
     FileUtils.stubs(:cp_r)
+    FileUtils.stubs(:cp)
 
     run_command("init", "--bundle").tap do |output|
       assert_match /Config file already exists in config\/deploy.yml \(remove first to create a new one\)/, output
@@ -321,7 +327,7 @@ class CliMainTest < CliTestCase
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with do |command, line, options, verbosity:|
       command == "bin/audit_broadcast" &&
         line =~ /\A'\[[^\]]+\] message'\z/ &&
-        options[:env].keys == %w[ MRSK_RECORDED_AT MRSK_PERFORMER MRSK_EVENT ] &&
+        options[:env].keys == %w[ MRSK_RECORDED_AT MRSK_PERFORMER MRSK_VERSION MRSK_EVENT ] &&
         verbosity == :debug
     end.returns("Broadcast audit message: message")
 

test/commands/auditor_test.rb

@@ -1,19 +1,26 @@
 require "test_helper"
+require "active_support/testing/time_helpers"
 
 class CommandsAuditorTest < ActiveSupport::TestCase
+  include ActiveSupport::Testing::TimeHelpers
+
   setup do
+    freeze_time
+
     @config = {
       service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
       audit_broadcast_cmd: "bin/audit_broadcast"
     }
 
     @auditor = new_command
+    @performer = `whoami`.strip
+    @recorded_at = Time.now.utc.iso8601
   end
 
   test "record" do
     assert_equal [
       :echo,
-      "[#{@auditor.details[:recorded_at]}]", "[#{@auditor.details[:performer]}]",
+      "[#{@recorded_at}] [#{@performer}]",
       "app removed container",
       ">>", "mrsk-app-audit.log"
     ], @auditor.record("app removed container")
@@ -23,7 +30,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
     new_command(destination: "staging").tap do |auditor|
       assert_equal [
         :echo,
-        "[#{auditor.details[:recorded_at]}]", "[#{auditor.details[:performer]}]", "[#{auditor.details[:destination]}]",
+        "[#{@recorded_at}] [#{@performer}] [staging]",
         "app removed container",
         ">>", "mrsk-app-staging-audit.log"
       ], auditor.record("app removed container")
@@ -34,7 +41,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
     new_command(role: "web").tap do |auditor|
       assert_equal [
         :echo,
-        "[#{auditor.details[:recorded_at]}]", "[#{auditor.details[:performer]}]", "[#{auditor.details[:role]}]",
+        "[#{@recorded_at}] [#{@performer}] [web]",
         "app removed container",
         ">>", "mrsk-app-audit.log"
       ], auditor.record("app removed container")
@@ -44,7 +51,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
   test "record with arg details" do
     assert_equal [
       :echo,
-      "[#{@auditor.details[:recorded_at]}]", "[#{@auditor.details[:performer]}]", "[value]",
+      "[#{@recorded_at}] [#{@performer}] [value]",
       "app removed container",
       ">>", "mrsk-app-audit.log"
     ], @auditor.record("app removed container", detail: "value")
@@ -53,10 +60,11 @@ class CommandsAuditorTest < ActiveSupport::TestCase
   test "broadcast" do
     assert_equal [
       "bin/audit_broadcast",
-      "'[#{@auditor.details[:performer]}] [value] app removed container'",
+      "'[#{@performer}] [value] app removed container'",
       env: {
-        "MRSK_RECORDED_AT" => @auditor.details[:recorded_at],
+        "MRSK_RECORDED_AT" => @recorded_at,
-        "MRSK_PERFORMER" => @auditor.details[:performer],
+        "MRSK_PERFORMER" => @performer,
+        "MRSK_VERSION" => "123",
         "MRSK_EVENT" => "app removed container",
         "MRSK_DETAIL" => "value"
       }

test/commands/hook_test.rb

@@ -0,0 +1,32 @@
+require "test_helper"
+
+class CommandsHookTest < ActiveSupport::TestCase
+  include ActiveSupport::Testing::TimeHelpers
+
+  setup do
+    freeze_time
+
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
+    }
+
+    @performer = `whoami`.strip
+    @recorded_at = Time.now.utc.iso8601
+  end
+
+  test "run" do
+    assert_equal [
+      ".mrsk/hooks/foo",
+      { env: {
+        "MRSK_RECORDED_AT" => @recorded_at,
+        "MRSK_PERFORMER" => @performer,
+        "MRSK_VERSION" => "123" } }
+    ], new_command.run("foo")
+  end
+
+  private
+    def new_command
+      Mrsk::Commands::Hook.new(Mrsk::Configuration.new(@config, version: "123"))
+    end
+end

test/integration/docker-compose.yml

@@ -17,6 +17,8 @@ services:
     privileged: true
     build:
       context: docker/deployer
+    environment:
+      - TEST_ID=${TEST_ID}
     volumes:
       - ../..:/mrsk
       - shared:/shared

test/integration/docker/deployer/app/.mrsk/hooks/post-push

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "Built and pushed!"
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/post-push

test/integration/docker/deployer/app/.mrsk/hooks/pre-build

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "About to build and push..."
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-build

test/integration/integration_test.rb

@@ -3,6 +3,7 @@ require "test_helper"
 
 class IntegrationTest < ActiveSupport::TestCase
   setup do
+    ENV["TEST_ID"] = SecureRandom.hex
     docker_compose "up --build -d"
     wait_for_healthy
     setup_deployer
@@ -14,7 +15,7 @@ class IntegrationTest < ActiveSupport::TestCase
 
   private
     def docker_compose(*commands, capture: false, raise_on_error: true)
-      command = "docker compose #{commands.join(" ")}"
+      command = "TEST_ID=#{ENV["TEST_ID"]} docker compose #{commands.join(" ")}"
       succeeded = false
       if capture
         result = stdouted { succeeded = system("cd test/integration && #{command}") }
@@ -82,6 +83,13 @@ class IntegrationTest < ActiveSupport::TestCase
       assert_equal version, response.body.strip
     end
 
+    def assert_hooks_ran
+      [ "pre-build", "post-push" ].each do |hook|
+        file = "/tmp/#{ENV["TEST_ID"]}/#{hook}"
+        assert_match /File: #{file}/, deployer_exec("stat #{file}", capture: true)
+      end
+    end
+
     def wait_for_healthy(timeout: 20)
       timeout_at = Time.now + timeout
       while docker_compose("ps -a | tail -n +2 | grep -v '(healthy)' | wc -l", capture: true) != "0"

test/integration/main_test.rb

@@ -8,6 +8,8 @@ class MainTest < IntegrationTest
 
     mrsk :deploy
 
+    assert_hooks_ran
+
     assert_app_is_up version: first_version
 
     second_version = update_app_rev