From 340ed94fa9851b85df5fb0a1c9d3f4588b9d6ff9 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Fri, 5 May 2023 09:50:27 +0100
Subject: [PATCH 01/14] Make verify_local_dependencies private

We don't need to what it returns, it raises if there is a problem.

Move it out of the run_locally block to make it easier to add hooks.
---
 lib/mrsk/cli/build.rb  | 31 ++++++++++++++-----------------
 test/cli/build_test.rb | 35 +++++++++++++++--------------------
 2 files changed, 29 insertions(+), 37 deletions(-)

diff --git a/lib/mrsk/cli/build.rb b/lib/mrsk/cli/build.rb
index 0e6e9a7c..22dae998 100644
--- a/lib/mrsk/cli/build.rb
+++ b/lib/mrsk/cli/build.rb
@@ -14,11 +14,11 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
     with_lock do
       cli = self
 
+      verify_local_dependencies
+
       run_locally do
         begin
-          if cli.verify_local_dependencies
-            MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
-          end
+          MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
         rescue SSHKit::Command::Failed => e
           if e.message =~ /(no builder)|(no such file or directory)/
             error "Missing compatible builder, so creating a new one first"
@@ -82,21 +82,18 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
     end
   end
 
+  private
+    def verify_local_dependencies
+      run_locally do
+        begin
+          execute *MRSK.builder.ensure_local_dependencies_installed
+        rescue SSHKit::Command::Failed => e
+          build_error = e.message =~ /command not found/ ?
+            "Docker is not installed locally" :
+            "Docker buildx plugin is not installed locally"
 
-  desc "", "" # Really a private method, but needed to be invoked from #push
-  def verify_local_dependencies
-    run_locally do
-      begin
-        execute *MRSK.builder.ensure_local_dependencies_installed
-      rescue SSHKit::Command::Failed => e
-        build_error = e.message =~ /command not found/ ?
-          "Docker is not installed locally" :
-          "Docker buildx plugin is not installed locally"
-
-        raise BuildError, build_error
+          raise BuildError, build_error
+        end
       end
     end
-
-    true
-  end
 end
diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb
index 0835262c..9f13ba6e 100644
--- a/test/cli/build_test.rb
+++ b/test/cli/build_test.rb
@@ -9,17 +9,19 @@ class CliBuildTest < CliTestCase
   end
 
   test "push" do
-    Mrsk::Cli::Build.any_instance.stubs(:verify_local_dependencies).returns(true)
     run_command("push").tap do |output|
+      assert_match /docker --version && docker buildx version/, output
       assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder mrsk-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
     end
   end
 
   test "push without builder" do
     stub_locking
-    Mrsk::Cli::Build.any_instance.stubs(:verify_local_dependencies).returns(true)
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |arg| arg == :docker }
+      .with(:docker, "--version", "&&", :docker, :buildx, "version")
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |*args| args[0..1] == [:docker, :buildx] }
       .raises(SSHKit::Command::Failed.new("no builder"))
       .then
       .returns(true)
@@ -29,6 +31,16 @@ class CliBuildTest < CliTestCase
     end
   end
 
+  test "push with no buildx plugin" do
+    stub_locking
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with(:docker, "--version", "&&", :docker, :buildx, "version")
+      .raises(SSHKit::Command::Failed.new("no buildx"))
+
+    Mrsk::Commands::Builder.any_instance.stubs(:native_and_local?).returns(false)
+    assert_raises(Mrsk::Cli::Build::BuildError) { run_command("push") }
+  end
+
   test "pull" do
     run_command("pull").tap do |output|
       assert_match /docker image rm --force dhh\/app:999/, output
@@ -70,23 +82,6 @@ class CliBuildTest < CliTestCase
     end
   end
 
-  test "verify local dependencies" do
-    Mrsk::Commands::Builder.any_instance.stubs(:name).returns("remote".inquiry)
-
-    run_command("verify_local_dependencies").tap do |output|
-      assert_match /docker --version && docker buildx version/, output
-    end
-  end
-
-  test "verify local dependencies with no buildx plugin" do
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, "--version", "&&", :docker, :buildx, "version")
-      .raises(SSHKit::Command::Failed.new("no buildx"))
-
-    Mrsk::Commands::Builder.any_instance.stubs(:native_and_local?).returns(false)
-    assert_raises(Mrsk::Cli::Build::BuildError) { run_command("verify_local_dependencies") }
-  end
-
   private
     def run_command(*command)
       stdouted { Mrsk::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }

From 58c1096a90abba9715fc6318902e599dceb00567 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Fri, 5 May 2023 15:08:28 +0100
Subject: [PATCH 02/14] MRSK hooks

Adds hooks to MRSK. Currently just two hooks, pre-build and post-push.

We could break the build and push into two separate commands if we
found the need for post-build and/or pre-push hooks.

Hooks are stored in `.mrsk/hooks`. Running `mrsk init` will now create
that folder and add sample hook scripts.

Hooks returning non-zero exit codes will abort the current command.

Further potential work here:
- We could replace the audit broadcast command with a
post-deploy/post-rollback hook or similar
- Maybe provide pre-command/post-command hooks that run after every
mrsk invocation
- Also look for hooks in `~/.mrsk/hooks`
---
 README.md                                     | 13 +++++
 lib/mrsk/cli.rb                               |  1 +
 lib/mrsk/cli/base.rb                          | 12 +++++
 lib/mrsk/cli/build.rb                         |  2 +
 lib/mrsk/cli/main.rb                          |  8 +++
 .../templates/sample_hooks/post-push.sample   | 20 ++++++++
 .../templates/sample_hooks/pre-build.sample   | 49 +++++++++++++++++++
 lib/mrsk/commander.rb                         | 12 +++--
 lib/mrsk/commands/auditor.rb                  | 33 ++-----------
 lib/mrsk/commands/base.rb                     |  4 ++
 lib/mrsk/commands/hook.rb                     | 14 ++++++
 lib/mrsk/tags.rb                              | 34 +++++++++++++
 test/cli/app_test.rb                          | 15 +++---
 test/cli/build_test.rb                        | 23 +++++++++
 test/cli/cli_test_case.rb                     | 13 +++++
 test/cli/main_test.rb                         | 16 ++++--
 test/commands/auditor_test.rb                 | 22 ++++++---
 test/commands/hook_test.rb                    | 32 ++++++++++++
 test/integration/docker-compose.yml           |  2 +
 .../docker/deployer/app/.mrsk/hooks/post-push |  3 ++
 .../docker/deployer/app/.mrsk/hooks/pre-build |  3 ++
 test/integration/integration_test.rb          | 10 +++-
 test/integration/main_test.rb                 |  2 +
 23 files changed, 292 insertions(+), 51 deletions(-)
 create mode 100755 lib/mrsk/cli/templates/sample_hooks/post-push.sample
 create mode 100755 lib/mrsk/cli/templates/sample_hooks/pre-build.sample
 create mode 100644 lib/mrsk/commands/hook.rb
 create mode 100644 lib/mrsk/tags.rb
 create mode 100644 test/commands/hook_test.rb
 create mode 100755 test/integration/docker/deployer/app/.mrsk/hooks/post-push
 create mode 100755 test/integration/docker/deployer/app/.mrsk/hooks/pre-build

diff --git a/README.md b/README.md
index 9203cac2..6dfa26ea 100644
--- a/README.md
+++ b/README.md
@@ -902,6 +902,19 @@ When `limit` is specified, containers will be booted on, at most, `limit` hosts
 
 These settings only apply when booting containers (using `mrsk deploy`, or `mrsk app boot`). For other commands, MRSK continues to run commands in parallel across all hosts.
 
+## Hooks
+
+You can run custom scripts at specific points with hooks.
+
+Hooks should be stored in the .mrsk/hooks folder. Running mrsk init will build that folder and add some sample scripts.
+
+If the script returns a non-zero exit code the command will be aborted.
+
+There are currently two hooks:
+
+- pre-build
+- post-push
+
 ## Stage of development
 
 This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com).
diff --git a/lib/mrsk/cli.rb b/lib/mrsk/cli.rb
index c974a814..bc541d06 100644
--- a/lib/mrsk/cli.rb
+++ b/lib/mrsk/cli.rb
@@ -1,5 +1,6 @@
 module Mrsk::Cli
   class LockError < StandardError; end
+  class HookError < StandardError; end
 end
 
 # SSHKit uses instance eval, so we need a global const for ergonomics
diff --git a/lib/mrsk/cli/base.rb b/lib/mrsk/cli/base.rb
index 7a261dfe..182ca392 100644
--- a/lib/mrsk/cli/base.rb
+++ b/lib/mrsk/cli/base.rb
@@ -134,5 +134,17 @@ module Mrsk::Cli
           MRSK.hold_lock_on_error = false
         end
       end
+
+      def run_hook(hook)
+        run_locally do
+          if MRSK.hook.hook_exists?(hook)
+            begin
+              MRSK.with_verbosity(:debug) { execute(*MRSK.hook.run(hook)) }
+            rescue SSHKit::Command::Failed
+              raise HookError.new("Hook `#{hook}` failed")
+            end
+          end
+        end
+      end
   end
 end
diff --git a/lib/mrsk/cli/build.rb b/lib/mrsk/cli/build.rb
index 22dae998..ad049ec9 100644
--- a/lib/mrsk/cli/build.rb
+++ b/lib/mrsk/cli/build.rb
@@ -16,6 +16,7 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
 
       verify_local_dependencies
 
+      run_hook("pre-build")
       run_locally do
         begin
           MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
@@ -31,6 +32,7 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
           end
         end
       end
+      run_hook("post-push")
     end
   end
 
diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb
index a3d129d2..5e8087f8 100644
--- a/lib/mrsk/cli/main.rb
+++ b/lib/mrsk/cli/main.rb
@@ -132,6 +132,14 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       puts "Created .env file"
     end
 
+    unless (hooks_dir = Pathname.new(File.expand_path(".mrsk/hooks"))).exist?
+      hooks_dir.mkpath
+      Pathname.new(File.expand_path("templates/sample_hooks", __dir__)).each_child do |sample_hook|
+        FileUtils.cp sample_hook, hooks_dir
+      end
+      puts "Created sample hooks in .mrsk/hooks"
+    end
+
     if options[:bundle]
       if (binstub = Pathname.new(File.expand_path("bin/mrsk"))).exist?
         puts "Binstub already exists in bin/mrsk (remove first to create a new one)"
diff --git a/lib/mrsk/cli/templates/sample_hooks/post-push.sample b/lib/mrsk/cli/templates/sample_hooks/post-push.sample
new file mode 100755
index 00000000..a5ca5d27
--- /dev/null
+++ b/lib/mrsk/cli/templates/sample_hooks/post-push.sample
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# A sample post-push hook
+#
+# Checks:
+# 1. We have a clean checkout
+#
+# These environment variables are available:
+# MRSK_RECORDED_AT
+# MRSK_PERFORMER
+# MRSK_VERSION
+# MRSK_DESTINATION (if set)
+
+if [ -n "$(git status --porcelain)" ]; then
+  echo "Git checkout is not clean, aborting..."
+  git status --porcelain >&2
+  exit 1
+fi
+
+exit 0
diff --git a/lib/mrsk/cli/templates/sample_hooks/pre-build.sample b/lib/mrsk/cli/templates/sample_hooks/pre-build.sample
new file mode 100755
index 00000000..0d12be7e
--- /dev/null
+++ b/lib/mrsk/cli/templates/sample_hooks/pre-build.sample
@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# A sample pre-build hook
+#
+# Checks:
+# 1. We have a clean checkout
+# 2. A remote is configured
+# 3. The branch has been pushed to the remote
+# 4. The version we are deploying matches the remote
+#
+# These environment variables are available:
+# MRSK_RECORDED_AT
+# MRSK_PERFORMER
+# MRSK_VERSION
+# MRSK_DESTINATION (if set)
+
+if [ -n "$(git status --porcelain)" ]; then
+  echo "Git checkout is not clean, aborting..." >&2
+  git status --porcelain >&2
+  exit 1
+fi
+
+first_remote=$(git remote)
+
+if [ -z "$first_remote" ]; then
+  echo "No git remote set, aborting..." >&2
+  exit 1
+fi
+
+current_branch=$(git branch --show-current)
+
+if [ -z "$current_branch" ]; then
+  echo "No git remote set, aborting..." >&2
+  exit 1
+fi
+
+remote_head=$(git ls-remote $first_remote --tags $current_branch | cut -f1)
+
+if [ -z "$remote_head" ]; then
+  echo "Branch not pushed to remote, aborting..." >&2
+  exit 1
+fi
+
+if [ "$MRSK_VERSION" != "$remote_head" ]; then
+  echo "Version ($MRSK_VERSION) does not match remote HEAD ($remote_head), aborting..." >&2
+  exit 1
+fi
+
+exit 0
diff --git a/lib/mrsk/commander.rb b/lib/mrsk/commander.rb
index 4ef45f0a..bdfef962 100644
--- a/lib/mrsk/commander.rb
+++ b/lib/mrsk/commander.rb
@@ -100,6 +100,14 @@ class Mrsk::Commander
     @healthcheck ||= Mrsk::Commands::Healthcheck.new(config)
   end
 
+  def hook
+    @hook ||= Mrsk::Commands::Hook.new(config)
+  end
+
+  def lock
+    @lock ||= Mrsk::Commands::Lock.new(config)
+  end
+
   def prune
     @prune ||= Mrsk::Commands::Prune.new(config)
   end
@@ -112,10 +120,6 @@ class Mrsk::Commander
     @traefik ||= Mrsk::Commands::Traefik.new(config)
   end
 
-  def lock
-    @lock ||= Mrsk::Commands::Lock.new(config)
-  end
-
   def with_verbosity(level)
     old_level = self.verbosity
 
diff --git a/lib/mrsk/commands/auditor.rb b/lib/mrsk/commands/auditor.rb
index 69ee03cd..032c2725 100644
--- a/lib/mrsk/commands/auditor.rb
+++ b/lib/mrsk/commands/auditor.rb
@@ -1,24 +1,23 @@
-require "time"
-
 class Mrsk::Commands::Auditor < Mrsk::Commands::Base
   attr_reader :details
 
   def initialize(config, **details)
     super(config)
-    @details = default_details.merge(details)
+    @details = details
   end
 
   # Runs remotely
   def record(line, **details)
     append \
-      [ :echo, *audit_tags(**details), line ],
+      [ :echo, audit_tags(**details).except(:version).to_s, line ],
       audit_log_file
   end
 
   # Runs locally
   def broadcast(line, **details)
     if broadcast_cmd = config.audit_broadcast_cmd
-      [ broadcast_cmd, *broadcast_args(line, **details), env: env_for(event: line, **details) ]
+      tags = audit_tags(**details, event: line)
+      [ broadcast_cmd,  "'#{tags.except(:recorded_at, :event, :version)} #{line}'", env: tags.env ]
     end
   end
 
@@ -31,29 +30,7 @@ class Mrsk::Commands::Auditor < Mrsk::Commands::Base
       [ "mrsk", config.service, config.destination, "audit.log" ].compact.join("-")
     end
 
-    def default_details
-      { recorded_at: Time.now.utc.iso8601,
-        performer: `whoami`.chomp,
-        destination: config.destination }
-    end
-
     def audit_tags(**details)
-      tags_for **self.details.merge(details)
-    end
-
-    def broadcast_args(line, **details)
-      "'#{broadcast_tags(**details).join(" ")} #{line}'"
-    end
-
-    def broadcast_tags(**details)
-      tags_for **self.details.merge(details).except(:recorded_at)
-    end
-
-    def tags_for(**details)
-      details.compact.values.map { |value| "[#{value}]" }
-    end
-
-    def env_for(**details)
-      self.details.merge(details).compact.transform_keys { |detail| "MRSK_#{detail.upcase}" }
+      tags(**self.details, **details)
     end
 end
diff --git a/lib/mrsk/commands/base.rb b/lib/mrsk/commands/base.rb
index 81c8420c..391c25f2 100644
--- a/lib/mrsk/commands/base.rb
+++ b/lib/mrsk/commands/base.rb
@@ -53,5 +53,9 @@ module Mrsk::Commands
       def docker(*args)
         args.compact.unshift :docker
       end
+
+      def tags(**details)
+        Mrsk::Tags.from_config(config, **details)
+      end
   end
 end
diff --git a/lib/mrsk/commands/hook.rb b/lib/mrsk/commands/hook.rb
new file mode 100644
index 00000000..5082e20c
--- /dev/null
+++ b/lib/mrsk/commands/hook.rb
@@ -0,0 +1,14 @@
+class Mrsk::Commands::Hook < Mrsk::Commands::Base
+  def run(hook, **details)
+    [ ".mrsk/hooks/#{hook}", env: tags(**details).env ]
+  end
+
+  def hook_exists?(hook)
+    Pathname.new(hook_file(hook)).exist?
+  end
+
+  private
+    def hook_file(hook)
+      ".mrsk/hooks/#{hook}"
+    end
+end
diff --git a/lib/mrsk/tags.rb b/lib/mrsk/tags.rb
new file mode 100644
index 00000000..2e9e4cf5
--- /dev/null
+++ b/lib/mrsk/tags.rb
@@ -0,0 +1,34 @@
+require "time"
+
+class Mrsk::Tags
+  attr_reader :config, :tags
+
+  class << self
+    def from_config(config, **extra)
+      new(**default_tags(config), **extra)
+    end
+
+    def default_tags(config)
+      { recorded_at: Time.now.utc.iso8601,
+        performer: `whoami`.chomp,
+        destination: config.destination,
+        version: config.version }
+    end
+  end
+
+  def initialize(**tags)
+    @tags = tags.compact
+  end
+
+  def env
+    tags.transform_keys { |detail| "MRSK_#{detail.upcase}" }
+  end
+
+  def to_s
+    tags.values.map { |value| "[#{value}]" }.join(" ")
+  end
+
+  def except(*tags)
+    self.class.new(**self.tags.except(*tags))
+  end
+end
diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb
index f198da3a..5ac35909 100644
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -2,12 +2,7 @@ require_relative "cli_test_case"
 
 class CliAppTest < CliTestCase
   test "boot" do
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-    .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-    .returns("running") # health check
-
+    stub_running
     run_command("boot").tap do |output|
       assert_match "docker tag dhh/app:latest dhh/app:latest", output
       assert_match "docker run --detach --restart unless-stopped", output
@@ -183,4 +178,12 @@ class CliAppTest < CliTestCase
     def run_command(*command, config: :with_accessories)
       stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1"]) }
     end
+
+    def stub_running
+      SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
+
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+        .returns("running") # health check
+    end
 end
diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb
index 9f13ba6e..8d70df2d 100644
--- a/test/cli/build_test.rb
+++ b/test/cli/build_test.rb
@@ -41,6 +41,22 @@ class CliBuildTest < CliTestCase
     assert_raises(Mrsk::Cli::Build::BuildError) { run_command("push") }
   end
 
+  test "push pre-build hook failure" do
+    fail_hook("pre-build")
+
+    assert_raises(Mrsk::Cli::HookError) { run_command("push") }
+
+    assert @executions.none? { |args| args[0..2] == [:docker, :buildx, :build] }
+  end
+
+  test "push post-push hook failure" do
+    fail_hook("post-push")
+
+    assert_raises(Mrsk::Cli::HookError) { run_command("push") }
+
+    assert @executions.any? { |args| args[0..2] == [:docker, :buildx, :build] }
+  end
+
   test "pull" do
     run_command("pull").tap do |output|
       assert_match /docker image rm --force dhh\/app:999/, output
@@ -93,4 +109,11 @@ class CliBuildTest < CliTestCase
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
         .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" }
     end
+
+    def stub_dependency_checks
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with(:docker, "--version", "&&", :docker, :buildx, "version")
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| args[0..1] == [:docker, :buildx] }
+    end
 end
diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb
index f582cc75..999ec4ea 100644
--- a/test/cli/cli_test_case.rb
+++ b/test/cli/cli_test_case.rb
@@ -14,4 +14,17 @@ class CliTestCase < ActiveSupport::TestCase
     ENV.delete("MYSQL_ROOT_PASSWORD")
     ENV.delete("VERSION")
   end
+
+  private
+    def fail_hook(hook)
+      @executions = []
+      Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| @executions << args; args != [".mrsk/hooks/#{hook}"] }
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| args.first == ".mrsk/hooks/#{hook}" }
+        .raises(SSHKit::Command::Failed.new("failed"))
+    end
+
 end
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index cde31923..6406c3c2 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -235,9 +235,11 @@ class CliMainTest < CliTestCase
   end
 
   test "init" do
-    Pathname.any_instance.expects(:exist?).returns(false).twice
+    Pathname.any_instance.expects(:exist?).returns(false).times(3)
+    Pathname.any_instance.stubs(:mkpath)
     FileUtils.stubs(:mkdir_p)
     FileUtils.stubs(:cp_r)
+    FileUtils.stubs(:cp)
 
     run_command("init").tap do |output|
       assert_match /Created configuration file in config\/deploy.yml/, output
@@ -246,7 +248,7 @@ class CliMainTest < CliTestCase
   end
 
   test "init with existing config" do
-    Pathname.any_instance.expects(:exist?).returns(true).twice
+    Pathname.any_instance.expects(:exist?).returns(true).times(3)
 
     run_command("init").tap do |output|
       assert_match /Config file already exists in config\/deploy.yml \(remove first to create a new one\)/, output
@@ -254,9 +256,11 @@ class CliMainTest < CliTestCase
   end
 
   test "init with bundle option" do
-    Pathname.any_instance.expects(:exist?).returns(false).times(3)
+    Pathname.any_instance.expects(:exist?).returns(false).times(4)
+    Pathname.any_instance.stubs(:mkpath)
     FileUtils.stubs(:mkdir_p)
     FileUtils.stubs(:cp_r)
+    FileUtils.stubs(:cp)
 
     run_command("init", "--bundle").tap do |output|
       assert_match /Created configuration file in config\/deploy.yml/, output
@@ -269,9 +273,11 @@ class CliMainTest < CliTestCase
   end
 
   test "init with bundle option and existing binstub" do
-    Pathname.any_instance.expects(:exist?).returns(true).times(3)
+    Pathname.any_instance.expects(:exist?).returns(true).times(4)
+    Pathname.any_instance.stubs(:mkpath)
     FileUtils.stubs(:mkdir_p)
     FileUtils.stubs(:cp_r)
+    FileUtils.stubs(:cp)
 
     run_command("init", "--bundle").tap do |output|
       assert_match /Config file already exists in config\/deploy.yml \(remove first to create a new one\)/, output
@@ -321,7 +327,7 @@ class CliMainTest < CliTestCase
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with do |command, line, options, verbosity:|
       command == "bin/audit_broadcast" &&
         line =~ /\A'\[[^\]]+\] message'\z/ &&
-        options[:env].keys == %w[ MRSK_RECORDED_AT MRSK_PERFORMER MRSK_EVENT ] &&
+        options[:env].keys == %w[ MRSK_RECORDED_AT MRSK_PERFORMER MRSK_VERSION MRSK_EVENT ] &&
         verbosity == :debug
     end.returns("Broadcast audit message: message")
 
diff --git a/test/commands/auditor_test.rb b/test/commands/auditor_test.rb
index bcf62929..2586c08a 100644
--- a/test/commands/auditor_test.rb
+++ b/test/commands/auditor_test.rb
@@ -1,19 +1,26 @@
 require "test_helper"
+require "active_support/testing/time_helpers"
 
 class CommandsAuditorTest < ActiveSupport::TestCase
+  include ActiveSupport::Testing::TimeHelpers
+
   setup do
+    freeze_time
+
     @config = {
       service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
       audit_broadcast_cmd: "bin/audit_broadcast"
     }
 
     @auditor = new_command
+    @performer = `whoami`.strip
+    @recorded_at = Time.now.utc.iso8601
   end
 
   test "record" do
     assert_equal [
       :echo,
-      "[#{@auditor.details[:recorded_at]}]", "[#{@auditor.details[:performer]}]",
+      "[#{@recorded_at}] [#{@performer}]",
       "app removed container",
       ">>", "mrsk-app-audit.log"
     ], @auditor.record("app removed container")
@@ -23,7 +30,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
     new_command(destination: "staging").tap do |auditor|
       assert_equal [
         :echo,
-        "[#{auditor.details[:recorded_at]}]", "[#{auditor.details[:performer]}]", "[#{auditor.details[:destination]}]",
+        "[#{@recorded_at}] [#{@performer}] [staging]",
         "app removed container",
         ">>", "mrsk-app-staging-audit.log"
       ], auditor.record("app removed container")
@@ -34,7 +41,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
     new_command(role: "web").tap do |auditor|
       assert_equal [
         :echo,
-        "[#{auditor.details[:recorded_at]}]", "[#{auditor.details[:performer]}]", "[#{auditor.details[:role]}]",
+        "[#{@recorded_at}] [#{@performer}] [web]",
         "app removed container",
         ">>", "mrsk-app-audit.log"
       ], auditor.record("app removed container")
@@ -44,7 +51,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
   test "record with arg details" do
     assert_equal [
       :echo,
-      "[#{@auditor.details[:recorded_at]}]", "[#{@auditor.details[:performer]}]", "[value]",
+      "[#{@recorded_at}] [#{@performer}] [value]",
       "app removed container",
       ">>", "mrsk-app-audit.log"
     ], @auditor.record("app removed container", detail: "value")
@@ -53,10 +60,11 @@ class CommandsAuditorTest < ActiveSupport::TestCase
   test "broadcast" do
     assert_equal [
       "bin/audit_broadcast",
-      "'[#{@auditor.details[:performer]}] [value] app removed container'",
+      "'[#{@performer}] [value] app removed container'",
       env: {
-        "MRSK_RECORDED_AT" => @auditor.details[:recorded_at],
-        "MRSK_PERFORMER" => @auditor.details[:performer],
+        "MRSK_RECORDED_AT" => @recorded_at,
+        "MRSK_PERFORMER" => @performer,
+        "MRSK_VERSION" => "123",
         "MRSK_EVENT" => "app removed container",
         "MRSK_DETAIL" => "value"
       }
diff --git a/test/commands/hook_test.rb b/test/commands/hook_test.rb
new file mode 100644
index 00000000..6d1380fd
--- /dev/null
+++ b/test/commands/hook_test.rb
@@ -0,0 +1,32 @@
+require "test_helper"
+
+class CommandsHookTest < ActiveSupport::TestCase
+  include ActiveSupport::Testing::TimeHelpers
+
+  setup do
+    freeze_time
+
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
+    }
+
+    @performer = `whoami`.strip
+    @recorded_at = Time.now.utc.iso8601
+  end
+
+  test "run" do
+    assert_equal [
+      ".mrsk/hooks/foo",
+      { env: {
+        "MRSK_RECORDED_AT" => @recorded_at,
+        "MRSK_PERFORMER" => @performer,
+        "MRSK_VERSION" => "123" } }
+    ], new_command.run("foo")
+  end
+
+  private
+    def new_command
+      Mrsk::Commands::Hook.new(Mrsk::Configuration.new(@config, version: "123"))
+    end
+end
diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml
index a3b2db66..0a92d427 100644
--- a/test/integration/docker-compose.yml
+++ b/test/integration/docker-compose.yml
@@ -17,6 +17,8 @@ services:
     privileged: true
     build:
       context: docker/deployer
+    environment:
+      - TEST_ID=${TEST_ID}
     volumes:
       - ../..:/mrsk
       - shared:/shared
diff --git a/test/integration/docker/deployer/app/.mrsk/hooks/post-push b/test/integration/docker/deployer/app/.mrsk/hooks/post-push
new file mode 100755
index 00000000..d9f037da
--- /dev/null
+++ b/test/integration/docker/deployer/app/.mrsk/hooks/post-push
@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "Built and pushed!"
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/post-push
diff --git a/test/integration/docker/deployer/app/.mrsk/hooks/pre-build b/test/integration/docker/deployer/app/.mrsk/hooks/pre-build
new file mode 100755
index 00000000..0cf78d88
--- /dev/null
+++ b/test/integration/docker/deployer/app/.mrsk/hooks/pre-build
@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "About to build and push..."
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-build
diff --git a/test/integration/integration_test.rb b/test/integration/integration_test.rb
index eaf2252b..628b9da8 100644
--- a/test/integration/integration_test.rb
+++ b/test/integration/integration_test.rb
@@ -3,6 +3,7 @@ require "test_helper"
 
 class IntegrationTest < ActiveSupport::TestCase
   setup do
+    ENV["TEST_ID"] = SecureRandom.hex
     docker_compose "up --build -d"
     wait_for_healthy
     setup_deployer
@@ -14,7 +15,7 @@ class IntegrationTest < ActiveSupport::TestCase
 
   private
     def docker_compose(*commands, capture: false, raise_on_error: true)
-      command = "docker compose #{commands.join(" ")}"
+      command = "TEST_ID=#{ENV["TEST_ID"]} docker compose #{commands.join(" ")}"
       succeeded = false
       if capture
         result = stdouted { succeeded = system("cd test/integration && #{command}") }
@@ -82,6 +83,13 @@ class IntegrationTest < ActiveSupport::TestCase
       assert_equal version, response.body.strip
     end
 
+    def assert_hooks_ran
+      [ "pre-build", "post-push" ].each do |hook|
+        file = "/tmp/#{ENV["TEST_ID"]}/#{hook}"
+        assert_match /File: #{file}/, deployer_exec("stat #{file}", capture: true)
+      end
+    end
+
     def wait_for_healthy(timeout: 20)
       timeout_at = Time.now + timeout
       while docker_compose("ps -a | tail -n +2 | grep -v '(healthy)' | wc -l", capture: true) != "0"
diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb
index bd4c2dcc..2a7823d6 100644
--- a/test/integration/main_test.rb
+++ b/test/integration/main_test.rb
@@ -8,6 +8,8 @@ class MainTest < IntegrationTest
 
     mrsk :deploy
 
+    assert_hooks_ran
+
     assert_app_is_up version: first_version
 
     second_version = update_app_rev

From f3ec9f19c8618507e84bee25fbf0e13dad677e73 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Thu, 11 May 2023 10:48:11 +0100
Subject: [PATCH 03/14] Add debug for failed version checks

---
 test/integration/integration_test.rb | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/test/integration/integration_test.rb b/test/integration/integration_test.rb
index 628b9da8..fb08af7c 100644
--- a/test/integration/integration_test.rb
+++ b/test/integration/integration_test.rb
@@ -90,6 +90,18 @@ class IntegrationTest < ActiveSupport::TestCase
       end
     end
 
+    def assert_200(response)
+      code = response.code
+      if code != "200"
+        puts "Got response code #{code}, here are the traefik logs:"
+        mrsk :traefik, :logs
+        puts "And here are the load balancer logs"
+        docker_compose :logs, :load_balancer
+        puts "Tried to get the response code again and got #{app_response.code}"
+      end
+      assert_equal "200", code
+    end
+
     def wait_for_healthy(timeout: 20)
       timeout_at = Time.now + timeout
       while docker_compose("ps -a | tail -n +2 | grep -v '(healthy)' | wc -l", capture: true) != "0"

From 910f14e9c0c898a0c13d7fb80bbc36cb98b8c4ea Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Mon, 22 May 2023 11:09:13 +0100
Subject: [PATCH 04/14] Add configuration for hooks_path

---
 README.md                  |  2 ++
 lib/mrsk/commands/hook.rb  |  4 ++--
 lib/mrsk/configuration.rb  |  6 +++++-
 test/commands/hook_test.rb | 14 ++++++++++++--
 4 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 6dfa26ea..ed6afca2 100644
--- a/README.md
+++ b/README.md
@@ -908,6 +908,8 @@ You can run custom scripts at specific points with hooks.
 
 Hooks should be stored in the .mrsk/hooks folder. Running mrsk init will build that folder and add some sample scripts.
 
+You can change their location by setting `hooks_path` in the configuration file.
+
 If the script returns a non-zero exit code the command will be aborted.
 
 There are currently two hooks:
diff --git a/lib/mrsk/commands/hook.rb b/lib/mrsk/commands/hook.rb
index 5082e20c..912d4480 100644
--- a/lib/mrsk/commands/hook.rb
+++ b/lib/mrsk/commands/hook.rb
@@ -1,6 +1,6 @@
 class Mrsk::Commands::Hook < Mrsk::Commands::Base
   def run(hook, **details)
-    [ ".mrsk/hooks/#{hook}", env: tags(**details).env ]
+    [ hook_file(hook), env: tags(**details).env ]
   end
 
   def hook_exists?(hook)
@@ -9,6 +9,6 @@ class Mrsk::Commands::Hook < Mrsk::Commands::Base
 
   private
     def hook_file(hook)
-      ".mrsk/hooks/#{hook}"
+      "#{config.hooks_path}/#{hook}"
     end
 end
diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb
index f4837079..c21c0149 100644
--- a/lib/mrsk/configuration.rb
+++ b/lib/mrsk/configuration.rb
@@ -6,7 +6,7 @@ require "erb"
 require "net/ssh/proxy/jump"
 
 class Mrsk::Configuration
-  delegate :service, :image, :servers, :env, :labels, :registry, :builder, :stop_wait_time, to: :raw_config, allow_nil: true
+  delegate :service, :image, :servers, :env, :labels, :registry, :builder, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
   delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Mrsk::Utils
 
   attr_accessor :destination
@@ -197,6 +197,10 @@ class Mrsk::Configuration
     raw_config.traefik || {}
   end
 
+  def hooks_path
+    raw_config.hooks_path || ".mrsk/hooks"
+  end
+
   private
     # Will raise ArgumentError if any required config keys are missing
     def ensure_required_keys_present
diff --git a/test/commands/hook_test.rb b/test/commands/hook_test.rb
index 6d1380fd..f88d87b3 100644
--- a/test/commands/hook_test.rb
+++ b/test/commands/hook_test.rb
@@ -25,8 +25,18 @@ class CommandsHookTest < ActiveSupport::TestCase
     ], new_command.run("foo")
   end
 
+  test "run with custom hooks_path" do
+    assert_equal [
+      "custom/hooks/path/foo",
+      { env: {
+        "MRSK_RECORDED_AT" => @recorded_at,
+        "MRSK_PERFORMER" => @performer,
+        "MRSK_VERSION" => "123" } }
+    ], new_command(hooks_path: "custom/hooks/path").run("foo")
+  end
+
   private
-    def new_command
-      Mrsk::Commands::Hook.new(Mrsk::Configuration.new(@config, version: "123"))
+    def new_command(**extra_config)
+      Mrsk::Commands::Hook.new(Mrsk::Configuration.new(@config.merge(**extra_config), version: "123"))
     end
 end

From 5ab630cb038535398aa3a9e6b1be51c2f8de86a3 Mon Sep 17 00:00:00 2001
From: David Heinemeier Hansson <dhh@hey.com>
Date: Mon, 22 May 2023 14:04:03 +0200
Subject: [PATCH 05/14] Style

---
 lib/mrsk/cli/build.rb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/mrsk/cli/build.rb b/lib/mrsk/cli/build.rb
index ad049ec9..f060cd10 100644
--- a/lib/mrsk/cli/build.rb
+++ b/lib/mrsk/cli/build.rb
@@ -15,8 +15,8 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
       cli = self
 
       verify_local_dependencies
+      run_hook "pre-build"
 
-      run_hook("pre-build")
       run_locally do
         begin
           MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
@@ -32,7 +32,8 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
           end
         end
       end
-      run_hook("post-push")
+
+      run_hook "post-push"
     end
   end
 

From 0bc1fbfb74f77e841979d9a1c59616ab346477e8 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Mon, 22 May 2023 14:49:04 +0100
Subject: [PATCH 06/14] Set max-concurrent-downloads to 1 to prevent timeouts

---
 test/integration/docker/deployer/boot.sh | 2 +-
 test/integration/docker/vm/boot.sh       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/integration/docker/deployer/boot.sh b/test/integration/docker/deployer/boot.sh
index b25f3334..77d6d1ea 100755
--- a/test/integration/docker/deployer/boot.sh
+++ b/test/integration/docker/deployer/boot.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
 
-dockerd &
+dockerd --max-concurrent-downloads 1 &
 
 exec sleep infinity
diff --git a/test/integration/docker/vm/boot.sh b/test/integration/docker/vm/boot.sh
index 81df5cdb..681a8a4e 100755
--- a/test/integration/docker/vm/boot.sh
+++ b/test/integration/docker/vm/boot.sh
@@ -4,6 +4,6 @@ while [ ! -f /root/.ssh/authorized_keys ]; do echo "Waiting for ssh keys"; sleep
 
 service ssh restart
 
-dockerd &
+dockerd --max-concurrent-downloads 1 &
 
 exec sleep infinity

From 38023fe5381a2a715bce9b38137b2149f9601fdc Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Mon, 22 May 2023 15:06:05 +0100
Subject: [PATCH 07/14] Remove post push hook

---
 README.md                                     |  3 +--
 lib/mrsk/cli/build.rb                         |  2 --
 .../templates/sample_hooks/post-push.sample   | 20 -------------------
 test/cli/build_test.rb                        |  8 --------
 .../docker/deployer/app/.mrsk/hooks/post-push |  3 ---
 test/integration/integration_test.rb          |  2 +-
 6 files changed, 2 insertions(+), 36 deletions(-)
 delete mode 100755 lib/mrsk/cli/templates/sample_hooks/post-push.sample
 delete mode 100755 test/integration/docker/deployer/app/.mrsk/hooks/post-push

diff --git a/README.md b/README.md
index ed6afca2..acb430bf 100644
--- a/README.md
+++ b/README.md
@@ -912,10 +912,9 @@ You can change their location by setting `hooks_path` in the configuration file.
 
 If the script returns a non-zero exit code the command will be aborted.
 
-There are currently two hooks:
+There is currently one hook:
 
 - pre-build
-- post-push
 
 ## Stage of development
 
diff --git a/lib/mrsk/cli/build.rb b/lib/mrsk/cli/build.rb
index f060cd10..9628c757 100644
--- a/lib/mrsk/cli/build.rb
+++ b/lib/mrsk/cli/build.rb
@@ -32,8 +32,6 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
           end
         end
       end
-
-      run_hook "post-push"
     end
   end
 
diff --git a/lib/mrsk/cli/templates/sample_hooks/post-push.sample b/lib/mrsk/cli/templates/sample_hooks/post-push.sample
deleted file mode 100755
index a5ca5d27..00000000
--- a/lib/mrsk/cli/templates/sample_hooks/post-push.sample
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/sh
-
-# A sample post-push hook
-#
-# Checks:
-# 1. We have a clean checkout
-#
-# These environment variables are available:
-# MRSK_RECORDED_AT
-# MRSK_PERFORMER
-# MRSK_VERSION
-# MRSK_DESTINATION (if set)
-
-if [ -n "$(git status --porcelain)" ]; then
-  echo "Git checkout is not clean, aborting..."
-  git status --porcelain >&2
-  exit 1
-fi
-
-exit 0
diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb
index 8d70df2d..ccf8e370 100644
--- a/test/cli/build_test.rb
+++ b/test/cli/build_test.rb
@@ -49,14 +49,6 @@ class CliBuildTest < CliTestCase
     assert @executions.none? { |args| args[0..2] == [:docker, :buildx, :build] }
   end
 
-  test "push post-push hook failure" do
-    fail_hook("post-push")
-
-    assert_raises(Mrsk::Cli::HookError) { run_command("push") }
-
-    assert @executions.any? { |args| args[0..2] == [:docker, :buildx, :build] }
-  end
-
   test "pull" do
     run_command("pull").tap do |output|
       assert_match /docker image rm --force dhh\/app:999/, output
diff --git a/test/integration/docker/deployer/app/.mrsk/hooks/post-push b/test/integration/docker/deployer/app/.mrsk/hooks/post-push
deleted file mode 100755
index d9f037da..00000000
--- a/test/integration/docker/deployer/app/.mrsk/hooks/post-push
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-echo "Built and pushed!"
-mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/post-push
diff --git a/test/integration/integration_test.rb b/test/integration/integration_test.rb
index fb08af7c..bbaab9bc 100644
--- a/test/integration/integration_test.rb
+++ b/test/integration/integration_test.rb
@@ -84,7 +84,7 @@ class IntegrationTest < ActiveSupport::TestCase
     end
 
     def assert_hooks_ran
-      [ "pre-build", "post-push" ].each do |hook|
+      [ "pre-build" ].each do |hook|
         file = "/tmp/#{ENV["TEST_ID"]}/#{hook}"
         assert_match /File: #{file}/, deployer_exec("stat #{file}", capture: true)
       end

From 9fd184dc329de3ea36e6e23558611e34b17b6dd0 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Tue, 23 May 2023 08:45:25 +0100
Subject: [PATCH 08/14] Add post-deploy and post-rollback hooks

These replace the custom audit_broadcast_cmd code. An additional env
variable MRSK_RUNTIME is passed to them.

The audit broadcast after booting an accessory has been removed.
---
 README.md                                     | 70 ++++++++-----------
 lib/mrsk/cli/accessory.rb                     |  2 -
 lib/mrsk/cli/base.rb                          |  8 +--
 lib/mrsk/cli/main.rb                          | 19 ++---
 lib/mrsk/cli/templates/deploy.yml             |  4 --
 .../templates/sample_hooks/post-deploy.sample | 18 +++++
 .../sample_hooks/post-rollback.sample         | 18 +++++
 lib/mrsk/commands/auditor.rb                  |  8 ---
 lib/mrsk/configuration.rb                     |  4 --
 test/cli/build_test.rb                        |  7 --
 test/cli/cli_test_case.rb                     | 15 ++++
 test/cli/main_test.rb                         | 27 ++++---
 test/commands/auditor_test.rb                 | 16 +----
 test/fixtures/deploy_simple.yml               |  1 -
 .../deployer/app/.mrsk/hooks/post-deploy      |  3 +
 .../deployer/app/.mrsk/hooks/post-rollback    |  3 +
 test/integration/integration_test.rb          |  6 +-
 test/integration/main_test.rb                 | 10 +--
 18 files changed, 117 insertions(+), 122 deletions(-)
 create mode 100644 lib/mrsk/cli/templates/sample_hooks/post-deploy.sample
 create mode 100644 lib/mrsk/cli/templates/sample_hooks/post-rollback.sample
 create mode 100755 test/integration/docker/deployer/app/.mrsk/hooks/post-deploy
 create mode 100755 test/integration/docker/deployer/app/.mrsk/hooks/post-rollback

diff --git a/README.md b/README.md
index acb430bf..b5e9b6bf 100644
--- a/README.md
+++ b/README.md
@@ -668,43 +668,6 @@ servers:
 
 This assumes the Cron settings are stored in `config/crontab`.
 
-### Using audit broadcasts
-
-If you'd like to broadcast audits of deploys, rollbacks, etc to a chatroom or elsewhere, you can configure the `audit_broadcast_cmd` setting with the path to a bin file that will be passed the audit line as the first argument:
-
-```yaml
-audit_broadcast_cmd:
-  bin/audit_broadcast
-```
-
-The broadcast command could look something like:
-
-```bash
-#!/usr/bin/env bash
-curl -q -d content="[My App] ${1}" https://3.basecamp.com/XXXXX/integrations/XXXXX/buckets/XXXXX/chats/XXXXX/lines
-```
-
-That'll post a line like follows to a preconfigured chatbot in Basecamp:
-
-```
-[My App] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de
-```
-
-`MRSK_*` environment variables are available to the broadcast command for
-fine-grained audit reporting, e.g. for triggering deployment reports or
-firing a JSON webhook. These variables include:
-- `MRSK_RECORDED_AT` - UTC timestamp in ISO 8601 format, e.g. `2023-04-14T17:07:31Z`
-- `MRSK_PERFORMER` - the local user performing the command (from `whoami`)
-- `MRSK_MESSAGE` - the full audit message, e.g. "Deployed app@150b24f"
-- `MRSK_DESTINATION` - optional: destination, e.g. "staging"
-- `MRSK_ROLE` - optional: role targeted, e.g. "web"
-
-Use `mrsk broadcast` to test and troubleshoot your broadcast command:
-
-```bash
-mrsk broadcast -m "test audit message"
-```
-
 ### Healthcheck
 
 MRSK uses Docker healtchecks to check the health of your application during deployment. Traefik uses this same healthcheck status to determine when a container is ready to receive traffic.
@@ -912,9 +875,38 @@ You can change their location by setting `hooks_path` in the configuration file.
 
 If the script returns a non-zero exit code the command will be aborted.
 
-There is currently one hook:
+`MRSK_*` environment variables are available to the hooks command for
+fine-grained audit reporting, e.g. for triggering deployment reports or
+firing a JSON webhook. These variables include:
+- `MRSK_RECORDED_AT` - UTC timestamp in ISO 8601 format, e.g. `2023-04-14T17:07:31Z`
+- `MRSK_PERFORMER` - the local user performing the command (from `whoami`)
+- `MRSK_MESSAGE` - the full audit message, e.g. "Deployed app@150b24f"
+- `MRSK_DESTINATION` - optional: destination, e.g. "staging"
+- `MRSK_ROLE` - optional: role targeted, e.g. "web"
 
-- pre-build
+There are three hooks:
+
+1. pre-build
+Used for pre-build checks - e.g. there are no uncommitted changes or that CI has passed.
+
+2. post-deploy and post-rollback
+
+These two hooks are also passed a `MRSK_RUNTIME` env variable.
+
+This could be used to broadcast a deployment message, or register the new version with an APM.
+
+The command could look something like:
+
+```bash
+#!/usr/bin/env bash
+curl -q -d content="[My App] ${MRSK_PERFORMER} Rolled back to version ${MRSK_VERSION}" https://3.basecamp.com/XXXXX/integrations/XXXXX/buckets/XXXXX/chats/XXXXX/lines
+```
+
+That'll post a line like follows to a preconfigured chatbot in Basecamp:
+
+```
+[My App] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de
+```
 
 ## Stage of development
 
diff --git a/lib/mrsk/cli/accessory.rb b/lib/mrsk/cli/accessory.rb
index c34a3902..1343a5f2 100644
--- a/lib/mrsk/cli/accessory.rb
+++ b/lib/mrsk/cli/accessory.rb
@@ -14,8 +14,6 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
             execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug
             execute *accessory.run
           end
-
-          audit_broadcast "Booted accessory #{name}" unless options[:skip_broadcast]
         end
       end
     end
diff --git a/lib/mrsk/cli/base.rb b/lib/mrsk/cli/base.rb
index 182ca392..cfe826a5 100644
--- a/lib/mrsk/cli/base.rb
+++ b/lib/mrsk/cli/base.rb
@@ -72,10 +72,6 @@ module Mrsk::Cli
         puts "  Finished all in #{sprintf("%.1f seconds", runtime)}"
       end
 
-      def audit_broadcast(line)
-        run_locally { execute *MRSK.auditor.broadcast(line), verbosity: :debug }
-      end
-
       def with_lock
         if MRSK.holding_lock?
           yield
@@ -135,11 +131,11 @@ module Mrsk::Cli
         end
       end
 
-      def run_hook(hook)
+      def run_hook(hook, **details)
         run_locally do
           if MRSK.hook.hook_exists?(hook)
             begin
-              MRSK.with_verbosity(:debug) { execute(*MRSK.hook.run(hook)) }
+              MRSK.with_verbosity(:debug) { execute(*MRSK.hook.run(hook, **details)) }
             rescue SSHKit::Command::Failed
               raise HookError.new("Hook `#{hook}` failed")
             end
diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb
index 5e8087f8..c6061a33 100644
--- a/lib/mrsk/cli/main.rb
+++ b/lib/mrsk/cli/main.rb
@@ -44,7 +44,7 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       end
     end
 
-    audit_broadcast "Deployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
+    run_hook "post-deploy", runtime: runtime.round
   end
 
   desc "redeploy", "Deploy app to servers without bootstrapping servers, starting Traefik, pruning, and registry login"
@@ -72,13 +72,15 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       end
     end
 
-    audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
+    run_hook "post-deploy", runtime: runtime.round
   end
 
   desc "rollback [VERSION]", "Rollback app to VERSION"
   def rollback(version)
-    with_lock do
-      invoke_options = deploy_options
+    rolled_back = false
+    runtime = print_runtime do
+      with_lock do
+        invoke_options = deploy_options
 
       MRSK.config.version = version
       old_version = nil
@@ -86,7 +88,7 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       if container_available?(version)
         invoke "mrsk:cli:app:boot", [], invoke_options.merge(version: version)
 
-        audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast]
+        run_hook "post-deploy", runtime: runtime.round
       else
         say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
       end
@@ -180,13 +182,6 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
     end
   end
 
-  desc "broadcast", "Broadcast an audit message"
-  option :message, aliases: "-m", type: :string, desc: "Audit message", required: true
-  def broadcast
-    say "Broadcast: #{options[:message]}", :magenta
-    audit_broadcast options[:message]
-  end
-
   desc "version", "Show MRSK version"
   def version
     puts Mrsk::VERSION
diff --git a/lib/mrsk/cli/templates/deploy.yml b/lib/mrsk/cli/templates/deploy.yml
index 7d710632..45b987f5 100644
--- a/lib/mrsk/cli/templates/deploy.yml
+++ b/lib/mrsk/cli/templates/deploy.yml
@@ -25,10 +25,6 @@ registry:
 #   secret:
 #     - RAILS_MASTER_KEY
 
-# Call a broadcast command on deploys.
-# audit_broadcast_cmd:
-#   bin/broadcast_to_bc
-
 # Use a different ssh user than root
 # ssh:
 #   user: app
diff --git a/lib/mrsk/cli/templates/sample_hooks/post-deploy.sample b/lib/mrsk/cli/templates/sample_hooks/post-deploy.sample
new file mode 100644
index 00000000..abc18ccb
--- /dev/null
+++ b/lib/mrsk/cli/templates/sample_hooks/post-deploy.sample
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# A sample post-deploy hook
+#
+# Checks:
+# 1. We have a clean checkout
+# 2. A remote is configured
+# 3. The branch has been pushed to the remote
+# 4. The version we are deploying matches the remote
+#
+# These environment variables are available:
+# MRSK_RECORDED_AT
+# MRSK_PERFORMER
+# MRSK_VERSION
+# MRSK_DESTINATION (if set)
+# MRSK_RUNTIME
+
+echo "$MRSK_PERFORMER deployed $MRSK_VERSION to $MRSK_DESTINATION in $MRSK_RUNTIME seconds"
diff --git a/lib/mrsk/cli/templates/sample_hooks/post-rollback.sample b/lib/mrsk/cli/templates/sample_hooks/post-rollback.sample
new file mode 100644
index 00000000..79e5680d
--- /dev/null
+++ b/lib/mrsk/cli/templates/sample_hooks/post-rollback.sample
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# A sample post-rollback hook
+#
+# Checks:
+# 1. We have a clean checkout
+# 2. A remote is configured
+# 3. The branch has been pushed to the remote
+# 4. The version we are deploying matches the remote
+#
+# These environment variables are available:
+# MRSK_RECORDED_AT
+# MRSK_PERFORMER
+# MRSK_VERSION
+# MRSK_DESTINATION (if set)
+# MRSK_RUNTIME
+
+echo "$MRSK_PERFORMER rolled back to $MRSK_VERSION on $MRSK_DESTINATION in $MRSK_RUNTIME seconds"
diff --git a/lib/mrsk/commands/auditor.rb b/lib/mrsk/commands/auditor.rb
index 032c2725..c11f63f8 100644
--- a/lib/mrsk/commands/auditor.rb
+++ b/lib/mrsk/commands/auditor.rb
@@ -13,14 +13,6 @@ class Mrsk::Commands::Auditor < Mrsk::Commands::Base
       audit_log_file
   end
 
-  # Runs locally
-  def broadcast(line, **details)
-    if broadcast_cmd = config.audit_broadcast_cmd
-      tags = audit_tags(**details, event: line)
-      [ broadcast_cmd,  "'#{tags.except(:recorded_at, :event, :version)} #{line}'", env: tags.env ]
-    end
-  end
-
   def reveal
     [ :tail, "-n", 50, audit_log_file ]
   end
diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb
index c21c0149..e78e45cb 100644
--- a/lib/mrsk/configuration.rb
+++ b/lib/mrsk/configuration.rb
@@ -157,10 +157,6 @@ class Mrsk::Configuration
   end
 
 
-  def audit_broadcast_cmd
-    raw_config.audit_broadcast_cmd
-  end
-
   def healthcheck
     { "path" => "/up", "port" => 3000, "max_attempts" => 7 }.merge(raw_config.healthcheck || {})
   end
diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb
index ccf8e370..ece83851 100644
--- a/test/cli/build_test.rb
+++ b/test/cli/build_test.rb
@@ -95,13 +95,6 @@ class CliBuildTest < CliTestCase
       stdouted { Mrsk::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
     end
 
-    def stub_locking
-      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock }
-      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" }
-    end
-
     def stub_dependency_checks
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
         .with(:docker, "--version", "&&", :docker, :buildx, "version")
diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb
index 999ec4ea..2455002e 100644
--- a/test/cli/cli_test_case.rb
+++ b/test/cli/cli_test_case.rb
@@ -27,4 +27,19 @@ class CliTestCase < ActiveSupport::TestCase
         .raises(SSHKit::Command::Failed.new("failed"))
     end
 
+    def ensure_hook_runs(hook)
+      Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |*args| args != [".mrsk/hooks/#{hook}"] }
+      SSHKit::Backend::Abstract.any_instance.expects(:execute)
+        .with { |*args| args.first == ".mrsk/hooks/#{hook}" }
+        .once
+    end
+
+    def stub_locking
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock }
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" }
+    end
 end
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index 6406c3c2..bac45eba 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -20,6 +20,9 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
 
+    stub_locking
+    ensure_hook_runs("post-deploy")
+
     run_command("deploy").tap do |output|
       assert_match /Log into image registry/, output
       assert_match /Build and push app image/, output
@@ -102,6 +105,9 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
 
+    stub_locking
+    ensure_hook_runs("post-deploy")
+
     run_command("redeploy").tap do |output|
       assert_match /Build and push app image/, output
       assert_match /Ensure app can pass healthcheck/, output
@@ -149,7 +155,6 @@ class CliMainTest < CliTestCase
         .returns("running").at_least_once # health check
     end
 
-
     run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
       assert_match "Start container with version 123", output
       assert_match "docker tag dhh/app:123 dhh/app:latest", output
@@ -180,6 +185,13 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "rollback runs post deploy hook" do
+    Mrsk::Cli::Main.any_instance.stubs(:container_available?).returns(true)
+
+    ensure_hook_runs("post-rollback")
+    run_command("rollback", "123")
+  end
+
   test "details" do
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:details")
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:details")
@@ -323,19 +335,6 @@ class CliMainTest < CliTestCase
     end
   end
 
-  test "broadcast" do
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with do |command, line, options, verbosity:|
-      command == "bin/audit_broadcast" &&
-        line =~ /\A'\[[^\]]+\] message'\z/ &&
-        options[:env].keys == %w[ MRSK_RECORDED_AT MRSK_PERFORMER MRSK_VERSION MRSK_EVENT ] &&
-        verbosity == :debug
-    end.returns("Broadcast audit message: message")
-
-    run_command("broadcast", "-m", "message").tap do |output|
-      assert_match "Broadcast: message", output
-    end
-  end
-
   test "version" do
     version = stdouted { Mrsk::Cli::Main.new.version }
     assert_equal Mrsk::VERSION, version
diff --git a/test/commands/auditor_test.rb b/test/commands/auditor_test.rb
index 2586c08a..1a1b03e5 100644
--- a/test/commands/auditor_test.rb
+++ b/test/commands/auditor_test.rb
@@ -8,8 +8,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
     freeze_time
 
     @config = {
-      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
-      audit_broadcast_cmd: "bin/audit_broadcast"
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ]
     }
 
     @auditor = new_command
@@ -57,19 +56,6 @@ class CommandsAuditorTest < ActiveSupport::TestCase
     ], @auditor.record("app removed container", detail: "value")
   end
 
-  test "broadcast" do
-    assert_equal [
-      "bin/audit_broadcast",
-      "'[#{@performer}] [value] app removed container'",
-      env: {
-        "MRSK_RECORDED_AT" => @recorded_at,
-        "MRSK_PERFORMER" => @performer,
-        "MRSK_VERSION" => "123",
-        "MRSK_EVENT" => "app removed container",
-        "MRSK_DETAIL" => "value"
-      }
-    ], @auditor.broadcast("app removed container", detail: "value")
-  end
 
   private
     def new_command(destination: nil, **details)
diff --git a/test/fixtures/deploy_simple.yml b/test/fixtures/deploy_simple.yml
index 6c3a34ba..520c138e 100644
--- a/test/fixtures/deploy_simple.yml
+++ b/test/fixtures/deploy_simple.yml
@@ -6,4 +6,3 @@ servers:
 registry:
   username: user
   password: pw
-audit_broadcast_cmd: "bin/audit_broadcast"
diff --git a/test/integration/docker/deployer/app/.mrsk/hooks/post-deploy b/test/integration/docker/deployer/app/.mrsk/hooks/post-deploy
new file mode 100755
index 00000000..0fcc6920
--- /dev/null
+++ b/test/integration/docker/deployer/app/.mrsk/hooks/post-deploy
@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "Deployed!"
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/post-deploy
diff --git a/test/integration/docker/deployer/app/.mrsk/hooks/post-rollback b/test/integration/docker/deployer/app/.mrsk/hooks/post-rollback
new file mode 100755
index 00000000..af1e7944
--- /dev/null
+++ b/test/integration/docker/deployer/app/.mrsk/hooks/post-rollback
@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "Rolled back!"
+mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/post-rollback
diff --git a/test/integration/integration_test.rb b/test/integration/integration_test.rb
index bbaab9bc..a35ae775 100644
--- a/test/integration/integration_test.rb
+++ b/test/integration/integration_test.rb
@@ -83,10 +83,10 @@ class IntegrationTest < ActiveSupport::TestCase
       assert_equal version, response.body.strip
     end
 
-    def assert_hooks_ran
-      [ "pre-build" ].each do |hook|
+    def assert_hooks_ran(*hooks)
+      hooks.each do |hook|
         file = "/tmp/#{ENV["TEST_ID"]}/#{hook}"
-        assert_match /File: #{file}/, deployer_exec("stat #{file}", capture: true)
+        assert_equal "removed '#{file}'", deployer_exec("rm -v #{file}", capture: true).strip
       end
     end
 
diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb
index 2a7823d6..866cd03b 100644
--- a/test/integration/main_test.rb
+++ b/test/integration/main_test.rb
@@ -7,23 +7,20 @@ class MainTest < IntegrationTest
     assert_app_is_down
 
     mrsk :deploy
-
-    assert_hooks_ran
-
     assert_app_is_up version: first_version
+    assert_hooks_ran "pre-build", "post-deploy"
 
     second_version = update_app_rev
 
     mrsk :redeploy
-
     assert_app_is_up version: second_version
+    assert_hooks_ran "pre-build", "post-deploy"
 
     mrsk :rollback, first_version
-
+    assert_hooks_ran "post-rollback"
     assert_app_is_up version: first_version
 
     details = mrsk :details, capture: true
-
     assert_match /Traefik Host: vm1/, details
     assert_match /Traefik Host: vm2/, details
     assert_match /App Host: vm1/, details
@@ -32,7 +29,6 @@ class MainTest < IntegrationTest
     assert_match /registry:4443\/app:#{first_version}/, details
 
     audit = mrsk :audit, capture: true
-
     assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
   end
 

From 258887a451bc588fb8239fdb5e1ba9db202bff8b Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Tue, 23 May 2023 09:11:24 +0100
Subject: [PATCH 09/14] Set sample hook permissions and preserve when copying

---
 lib/mrsk/cli/main.rb                                     | 2 +-
 lib/mrsk/cli/templates/sample_hooks/post-deploy.sample   | 0
 lib/mrsk/cli/templates/sample_hooks/post-rollback.sample | 0
 3 files changed, 1 insertion(+), 1 deletion(-)
 mode change 100644 => 100755 lib/mrsk/cli/templates/sample_hooks/post-deploy.sample
 mode change 100644 => 100755 lib/mrsk/cli/templates/sample_hooks/post-rollback.sample

diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb
index c6061a33..45f996a5 100644
--- a/lib/mrsk/cli/main.rb
+++ b/lib/mrsk/cli/main.rb
@@ -137,7 +137,7 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
     unless (hooks_dir = Pathname.new(File.expand_path(".mrsk/hooks"))).exist?
       hooks_dir.mkpath
       Pathname.new(File.expand_path("templates/sample_hooks", __dir__)).each_child do |sample_hook|
-        FileUtils.cp sample_hook, hooks_dir
+        FileUtils.cp sample_hook, hooks_dir, preserve: true
       end
       puts "Created sample hooks in .mrsk/hooks"
     end
diff --git a/lib/mrsk/cli/templates/sample_hooks/post-deploy.sample b/lib/mrsk/cli/templates/sample_hooks/post-deploy.sample
old mode 100644
new mode 100755
diff --git a/lib/mrsk/cli/templates/sample_hooks/post-rollback.sample b/lib/mrsk/cli/templates/sample_hooks/post-rollback.sample
old mode 100644
new mode 100755

From 3b695ae127a256ce2ca558d6f1c322c7a44f5a64 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Tue, 23 May 2023 10:46:17 +0100
Subject: [PATCH 10/14] Add service_version and add running hook message

---
 README.md                    |  1 +
 lib/mrsk/cli/base.rb         | 13 ++++++-------
 lib/mrsk/cli/main.rb         |  4 ----
 lib/mrsk/commands/auditor.rb |  2 +-
 lib/mrsk/tags.rb             |  7 ++++++-
 lib/mrsk/utils.rb            |  9 ++++++++-
 test/commands/hook_test.rb   |  6 ++++--
 7 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index b5e9b6bf..227d0ba2 100644
--- a/README.md
+++ b/README.md
@@ -881,6 +881,7 @@ firing a JSON webhook. These variables include:
 - `MRSK_RECORDED_AT` - UTC timestamp in ISO 8601 format, e.g. `2023-04-14T17:07:31Z`
 - `MRSK_PERFORMER` - the local user performing the command (from `whoami`)
 - `MRSK_MESSAGE` - the full audit message, e.g. "Deployed app@150b24f"
+- `MRSK_SERVICE_VERSION` - an abbreviated version (for use in messages)
 - `MRSK_DESTINATION` - optional: destination, e.g. "staging"
 - `MRSK_ROLE` - optional: role targeted, e.g. "web"
 
diff --git a/lib/mrsk/cli/base.rb b/lib/mrsk/cli/base.rb
index cfe826a5..038ac3f0 100644
--- a/lib/mrsk/cli/base.rb
+++ b/lib/mrsk/cli/base.rb
@@ -132,13 +132,12 @@ module Mrsk::Cli
       end
 
       def run_hook(hook, **details)
-        run_locally do
-          if MRSK.hook.hook_exists?(hook)
-            begin
-              MRSK.with_verbosity(:debug) { execute(*MRSK.hook.run(hook, **details)) }
-            rescue SSHKit::Command::Failed
-              raise HookError.new("Hook `#{hook}` failed")
-            end
+        if MRSK.hook.hook_exists?(hook)
+          say "Running the #{hook} hook...", :magenta
+          run_locally do
+            MRSK.with_verbosity(:debug) { execute *MRSK.hook.run(hook, **details) }
+          rescue SSHKit::Command::Failed
+            raise HookError.new("Hook `#{hook}` failed")
           end
         end
       end
diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb
index 45f996a5..d305ebb1 100644
--- a/lib/mrsk/cli/main.rb
+++ b/lib/mrsk/cli/main.rb
@@ -238,8 +238,4 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
     def deploy_options
       { "version" => MRSK.config.version }.merge(options.without("skip_push"))
     end
-
-    def service_version(version = MRSK.config.abbreviated_version)
-      [ MRSK.config.service, version ].compact.join("@")
-    end
 end
diff --git a/lib/mrsk/commands/auditor.rb b/lib/mrsk/commands/auditor.rb
index c11f63f8..5089fb4d 100644
--- a/lib/mrsk/commands/auditor.rb
+++ b/lib/mrsk/commands/auditor.rb
@@ -9,7 +9,7 @@ class Mrsk::Commands::Auditor < Mrsk::Commands::Base
   # Runs remotely
   def record(line, **details)
     append \
-      [ :echo, audit_tags(**details).except(:version).to_s, line ],
+      [ :echo, audit_tags(**details).except(:version, :service_version).to_s, line ],
       audit_log_file
   end
 
diff --git a/lib/mrsk/tags.rb b/lib/mrsk/tags.rb
index 2e9e4cf5..c134a8f7 100644
--- a/lib/mrsk/tags.rb
+++ b/lib/mrsk/tags.rb
@@ -12,7 +12,12 @@ class Mrsk::Tags
       { recorded_at: Time.now.utc.iso8601,
         performer: `whoami`.chomp,
         destination: config.destination,
-        version: config.version }
+        version: config.version,
+        service_version: service_version(config) }
+    end
+
+    def service_version(config)
+      [ config.service, config.abbreviated_version ].compact.join("@")
     end
   end
 
diff --git a/lib/mrsk/utils.rb b/lib/mrsk/utils.rb
index 62dc6d8d..e463f547 100644
--- a/lib/mrsk/utils.rb
+++ b/lib/mrsk/utils.rb
@@ -84,6 +84,13 @@ module Mrsk::Utils
 
   # Abbreviate a git revhash for concise display
   def abbreviate_version(version)
-    version[0...7] if version
+    if version
+      # Don't abbreviate <sha>_uncommitted_<etc>
+      if version.include?("_")
+        version
+      else
+        version[0...7]
+      end
+    end
   end
 end
diff --git a/test/commands/hook_test.rb b/test/commands/hook_test.rb
index f88d87b3..726ef7d8 100644
--- a/test/commands/hook_test.rb
+++ b/test/commands/hook_test.rb
@@ -21,7 +21,8 @@ class CommandsHookTest < ActiveSupport::TestCase
       { env: {
         "MRSK_RECORDED_AT" => @recorded_at,
         "MRSK_PERFORMER" => @performer,
-        "MRSK_VERSION" => "123" } }
+        "MRSK_VERSION" => "123",
+        "MRSK_SERVICE_VERSION" => "app@123" } }
     ], new_command.run("foo")
   end
 
@@ -31,7 +32,8 @@ class CommandsHookTest < ActiveSupport::TestCase
       { env: {
         "MRSK_RECORDED_AT" => @recorded_at,
         "MRSK_PERFORMER" => @performer,
-        "MRSK_VERSION" => "123" } }
+        "MRSK_VERSION" => "123",
+        "MRSK_SERVICE_VERSION" => "app@123" } }
     ], new_command(hooks_path: "custom/hooks/path").run("foo")
   end
 

From 004f1b04e6ba51c83b51dc18823fd08b66c7b2a2 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Tue, 23 May 2023 11:08:53 +0100
Subject: [PATCH 11/14] Remove the skip_broadcast option

---
 lib/mrsk/cli/base.rb  |  2 --
 test/cli/main_test.rb | 27 ++++++++++++++++++++++-----
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/lib/mrsk/cli/base.rb b/lib/mrsk/cli/base.rb
index 038ac3f0..6e158920 100644
--- a/lib/mrsk/cli/base.rb
+++ b/lib/mrsk/cli/base.rb
@@ -20,8 +20,6 @@ module Mrsk::Cli
     class_option :config_file, aliases: "-c", default: "config/deploy.yml", desc: "Path to config file"
     class_option :destination, aliases: "-d", desc: "Specify destination to be used for config file (staging -> deploy.staging.yml)"
 
-    class_option :skip_broadcast, aliases: "-B", type: :boolean, default: false, desc: "Skip audit broadcasts"
-
     def initialize(*)
       super
       load_envs
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index bac45eba..ed6d32b9 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -10,7 +10,7 @@ class CliMainTest < CliTestCase
   end
 
   test "deploy" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
@@ -34,7 +34,7 @@ class CliMainTest < CliTestCase
   end
 
   test "deploy with skip_push" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
@@ -83,8 +83,25 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "deploy errors during critical section leave lock in place" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
+
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options).raises(RuntimeError)
+
+    assert !MRSK.holding_lock?
+    assert_raises(RuntimeError) do
+      stderred { run_command("deploy") }
+    end
+    assert MRSK.holding_lock?
+  end
+
   test "deploy errors during outside section leave remove lock" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke)
       .with("mrsk:cli:registry:login", [], invoke_options)
@@ -98,7 +115,7 @@ class CliMainTest < CliTestCase
   end
 
   test "redeploy" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
@@ -115,7 +132,7 @@ class CliMainTest < CliTestCase
   end
 
   test "redeploy with skip_push" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)

From cc2b321d939a2de5e767f2ff217dc7a4cd27c8e4 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Tue, 23 May 2023 12:14:37 +0100
Subject: [PATCH 12/14] Combine post-deploy and post-rollback

---
 README.md                                      |  6 +++---
 .../sample_hooks/post-rollback.sample          | 18 ------------------
 test/cli/main_test.rb                          |  2 +-
 .../deployer/app/.mrsk/hooks/post-rollback     |  3 ---
 test/integration/main_test.rb                  |  2 +-
 5 files changed, 5 insertions(+), 26 deletions(-)
 delete mode 100755 lib/mrsk/cli/templates/sample_hooks/post-rollback.sample
 delete mode 100755 test/integration/docker/deployer/app/.mrsk/hooks/post-rollback

diff --git a/README.md b/README.md
index 227d0ba2..d0279942 100644
--- a/README.md
+++ b/README.md
@@ -885,14 +885,14 @@ firing a JSON webhook. These variables include:
 - `MRSK_DESTINATION` - optional: destination, e.g. "staging"
 - `MRSK_ROLE` - optional: role targeted, e.g. "web"
 
-There are three hooks:
+There are two hooks:
 
 1. pre-build
 Used for pre-build checks - e.g. there are no uncommitted changes or that CI has passed.
 
-2. post-deploy and post-rollback
+2. post-deploy - run after a deploy, redeploy or rollback
 
-These two hooks are also passed a `MRSK_RUNTIME` env variable.
+This hook is also passed a `MRSK_RUNTIME` env variable.
 
 This could be used to broadcast a deployment message, or register the new version with an APM.
 
diff --git a/lib/mrsk/cli/templates/sample_hooks/post-rollback.sample b/lib/mrsk/cli/templates/sample_hooks/post-rollback.sample
deleted file mode 100755
index 79e5680d..00000000
--- a/lib/mrsk/cli/templates/sample_hooks/post-rollback.sample
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/sh
-
-# A sample post-rollback hook
-#
-# Checks:
-# 1. We have a clean checkout
-# 2. A remote is configured
-# 3. The branch has been pushed to the remote
-# 4. The version we are deploying matches the remote
-#
-# These environment variables are available:
-# MRSK_RECORDED_AT
-# MRSK_PERFORMER
-# MRSK_VERSION
-# MRSK_DESTINATION (if set)
-# MRSK_RUNTIME
-
-echo "$MRSK_PERFORMER rolled back to $MRSK_VERSION on $MRSK_DESTINATION in $MRSK_RUNTIME seconds"
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index ed6d32b9..270e09b7 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -205,7 +205,7 @@ class CliMainTest < CliTestCase
   test "rollback runs post deploy hook" do
     Mrsk::Cli::Main.any_instance.stubs(:container_available?).returns(true)
 
-    ensure_hook_runs("post-rollback")
+    ensure_hook_runs("post-deploy")
     run_command("rollback", "123")
   end
 
diff --git a/test/integration/docker/deployer/app/.mrsk/hooks/post-rollback b/test/integration/docker/deployer/app/.mrsk/hooks/post-rollback
deleted file mode 100755
index af1e7944..00000000
--- a/test/integration/docker/deployer/app/.mrsk/hooks/post-rollback
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-echo "Rolled back!"
-mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/post-rollback
diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb
index 866cd03b..9fecf6b1 100644
--- a/test/integration/main_test.rb
+++ b/test/integration/main_test.rb
@@ -17,7 +17,7 @@ class MainTest < IntegrationTest
     assert_hooks_ran "pre-build", "post-deploy"
 
     mrsk :rollback, first_version
-    assert_hooks_ran "post-rollback"
+    assert_hooks_ran "post-deploy"
     assert_app_is_up version: first_version
 
     details = mrsk :details, capture: true

From f9cb87e55afb57118bf3195a889da7803c53177f Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Tue, 23 May 2023 14:10:38 +0100
Subject: [PATCH 13/14] Fixup rebase issues

---
 lib/mrsk/cli/main.rb  | 18 ++++++++++--------
 test/cli/main_test.rb | 30 +++++++++++++-----------------
 2 files changed, 23 insertions(+), 25 deletions(-)

diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb
index d305ebb1..8f078bdc 100644
--- a/lib/mrsk/cli/main.rb
+++ b/lib/mrsk/cli/main.rb
@@ -82,17 +82,19 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       with_lock do
         invoke_options = deploy_options
 
-      MRSK.config.version = version
-      old_version = nil
+        MRSK.config.version = version
+        old_version = nil
 
-      if container_available?(version)
-        invoke "mrsk:cli:app:boot", [], invoke_options.merge(version: version)
-
-        run_hook "post-deploy", runtime: runtime.round
-      else
-        say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
+        if container_available?(version)
+          invoke "mrsk:cli:app:boot", [], invoke_options.merge(version: version)
+          rolled_back = true
+        else
+          say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
+        end
       end
     end
+
+    run_hook "post-deploy", runtime: runtime.round if rolled_back
   end
 
   desc "details", "Show details about all containers"
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index 270e09b7..050becc5 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -83,23 +83,6 @@ class CliMainTest < CliTestCase
     end
   end
 
-  test "deploy errors during critical section leave lock in place" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
-
-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options)
-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options).raises(RuntimeError)
-
-    assert !MRSK.holding_lock?
-    assert_raises(RuntimeError) do
-      stderred { run_command("deploy") }
-    end
-    assert MRSK.holding_lock?
-  end
-
   test "deploy errors during outside section leave remove lock" do
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
 
@@ -205,7 +188,20 @@ class CliMainTest < CliTestCase
   test "rollback runs post deploy hook" do
     Mrsk::Cli::Main.any_instance.stubs(:container_available?).returns(true)
 
+    Mrsk::Utils::HealthcheckPoller.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("").at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
+      .returns("").at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running").at_least_once # health check
+
     ensure_hook_runs("post-deploy")
+
     run_command("rollback", "123")
   end
 

From 19f0f40adf444c4314af055a3815aeb4564b51d5 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Tue, 23 May 2023 15:56:47 +0100
Subject: [PATCH 14/14] Add skip_hooks option

---
 README.md             |  2 ++
 lib/mrsk/cli/base.rb  |  4 ++-
 test/cli/app_test.rb  |  2 +-
 test/cli/main_test.rb | 57 +++++++++++++++++++++----------------------
 4 files changed, 34 insertions(+), 31 deletions(-)

diff --git a/README.md b/README.md
index d0279942..ca550b15 100644
--- a/README.md
+++ b/README.md
@@ -909,6 +909,8 @@ That'll post a line like follows to a preconfigured chatbot in Basecamp:
 [My App] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de
 ```
 
+Set `--skip_hooks` to avoid running the hooks.
+
 ## Stage of development
 
 This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com).
diff --git a/lib/mrsk/cli/base.rb b/lib/mrsk/cli/base.rb
index 6e158920..395df4cc 100644
--- a/lib/mrsk/cli/base.rb
+++ b/lib/mrsk/cli/base.rb
@@ -20,6 +20,8 @@ module Mrsk::Cli
     class_option :config_file, aliases: "-c", default: "config/deploy.yml", desc: "Path to config file"
     class_option :destination, aliases: "-d", desc: "Specify destination to be used for config file (staging -> deploy.staging.yml)"
 
+    class_option :skip_hooks, aliases: "-H", type: :boolean, default: false, desc: "Don't run hooks"
+
     def initialize(*)
       super
       load_envs
@@ -130,7 +132,7 @@ module Mrsk::Cli
       end
 
       def run_hook(hook, **details)
-        if MRSK.hook.hook_exists?(hook)
+        if !options[:skip_hooks] && MRSK.hook.hook_exists?(hook)
           say "Running the #{hook} hook...", :magenta
           run_locally do
             MRSK.with_verbosity(:debug) { execute *MRSK.hook.run(hook, **details) }
diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb
index 5ac35909..806c8afe 100644
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -46,7 +46,7 @@ class CliAppTest < CliTestCase
   end
 
   test "boot errors leave lock in place" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
 
     Mrsk::Cli::App.any_instance.expects(:using_version).raises(RuntimeError)
 
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index 050becc5..b2884b4e 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -10,7 +10,7 @@ class CliMainTest < CliTestCase
   end
 
   test "deploy" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
@@ -20,8 +20,7 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
 
-    stub_locking
-    ensure_hook_runs("post-deploy")
+    Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
 
     run_command("deploy").tap do |output|
       assert_match /Log into image registry/, output
@@ -30,11 +29,12 @@ class CliMainTest < CliTestCase
       assert_match /Ensure app can pass healthcheck/, output
       assert_match /Detect stale containers/, output
       assert_match /Prune old containers and images/, output
+      assert_match /Running the post-deploy hook.../, output
     end
   end
 
   test "deploy with skip_push" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
@@ -84,7 +84,7 @@ class CliMainTest < CliTestCase
   end
 
   test "deploy errors during outside section leave remove lock" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke)
       .with("mrsk:cli:registry:login", [], invoke_options)
@@ -97,25 +97,41 @@ class CliMainTest < CliTestCase
     assert !MRSK.holding_lock?
   end
 
+  test "deploy with skipped hooks" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => true }
+
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
+
+    run_command("deploy", "--skip_hooks") do
+      refute_match /Running the post-deploy hook.../, output
+    end
+  end
+
   test "redeploy" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
 
-    stub_locking
-    ensure_hook_runs("post-deploy")
+    Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
 
     run_command("redeploy").tap do |output|
       assert_match /Build and push app image/, output
       assert_match /Ensure app can pass healthcheck/, output
+      assert_match /Running the post-deploy hook.../, output
     end
   end
 
   test "redeploy with skip_push" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
@@ -155,11 +171,14 @@ class CliMainTest < CliTestCase
         .returns("running").at_least_once # health check
     end
 
+    Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
+
     run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
       assert_match "Start container with version 123", output
       assert_match "docker tag dhh/app:123 dhh/app:latest", output
       assert_match "docker start app-web-123", output
       assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running"
+      assert_match "Running the post-deploy hook...", output
     end
   end
 
@@ -185,26 +204,6 @@ class CliMainTest < CliTestCase
     end
   end
 
-  test "rollback runs post deploy hook" do
-    Mrsk::Cli::Main.any_instance.stubs(:container_available?).returns(true)
-
-    Mrsk::Utils::HealthcheckPoller.stubs(:sleep)
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
-      .returns("").at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
-      .returns("").at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
-      .returns("running").at_least_once # health check
-
-    ensure_hook_runs("post-deploy")
-
-    run_command("rollback", "123")
-  end
-
   test "details" do
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:details")
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:details")