Bump version for 0.7.0

Add healthcheck before deploy

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    mrsk (0.6.4)
+    mrsk (0.7.0)
       activesupport (>= 7.0)
       dotenv (~> 2.8)
       sshkit (~> 1.21)

README.md

@@ -20,7 +20,7 @@ env:
     - RAILS_MASTER_KEY
 ```
 
-Then edit your `.env` file to add your registry password as `MRSK_REGISTRY_PASSWORD` (and your `RAILS_MASTER_KEY` for production with a Rails app).
+Then edit your `.env` file to add your registry password as `MRSK_REGISTRY_PASSWORD` (and your `RAILS_MASTER_KEY` for production with a Rails app). 
 
 Now you're ready to deploy to the servers:
 
@@ -37,9 +37,10 @@ This will:
 5. Push the image to the registry.
 6. Pull the image from the registry on the servers.
 7. Ensure Traefik is running and accepting traffic on port 80.
-8. Stop any containers running a previous versions of the app.
+8. Ensure your app responds with `200 OK` to `GET /up`.
-9. Start a new container with the version of the app that matches the current git version hash.
+9. Stop any containers running a previous versions of the app.
-10. Prune unused images and stopped containers to ensure servers don't fill up.
+10. Start a new container with the version of the app that matches the current git version hash.
+11. Prune unused images and stopped containers to ensure servers don't fill up.
 
 Voila! All the servers are now serving the app on port 80. If you're just running a single server, you're ready to go. If you're running multiple servers, you need to put a load balancer in front of them.
 
@@ -183,10 +184,10 @@ You can specialize the default Traefik rules by setting labels on the containers
 
 ```
 labels:
-  traefik.http.routers.hey.rule: '''Host(`app.hey.com`)'''
+  traefik.http.routers.hey.rule: Host(\`app.hey.com\`)
 ```
 
-Note: The extra quotes are needed to ensure the rule is passed in correctly!
+Note: The escaped backticks are needed to ensure the rule is passed in correctly and not treated as command substitution by Bash!
 
 This allows you to run multiple applications on the same server sharing the same Traefik instance and port.
 See https://doc.traefik.io/traefik/routing/routers/#rule for a full list of available routing rules.
@@ -345,6 +346,41 @@ This template can safely be checked into git. Then everyone deploying the app ca
 
 If you need separate env variables for different destinations, you can set them with `.env.destination.erb` for the template, which will generate `.env.staging` when run with `mrsk envify -d staging`.
 
+### Using audit broadcasts
+
+If you'd like to broadcast audits of deploys, rollbacks, etc to a chatroom or elsewhere, you can configure the `audit_broadcast_cmd` setting with the path to a bin file that reads the audit line from STDIN, and then does whatever with it:
+
+```yaml
+audit_broadcast_cmd:
+  bin/audit_broadcast
+```
+
+The broadcast command could look something like:
+
+```bash
+#!/usr/bin/env bash
+read
+curl -q -d content="[My app] ${REPLY}" https://3.basecamp.com/XXXXX/integrations/XXXXX/buckets/XXXXX/chats/XXXXX/lines
+```
+
+That'll post a line like follows to a preconfigured chatbot in Basecamp:
+
+```
+[My App] [2023-02-18 11:29:52] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de
+```
+
+### Using custom healthcheck path or port
+
+MRSK defaults to checking the health of your application again `/up` on port 3000. You can tailor both with the `healthcheck` setting:
+
+```yaml
+healthcheck:
+  path: /healthz
+  port: 4000
+```
+
+This will ensure your application is configured with a traefik label for the healthcheck against `/healthz` and that the pre-deploy healthcheck that MRSK performs is done against the same path on port 4000.
+
 ## Commands
 
 ### Running commands on servers

lib/mrsk/cli/accessory.rb

@@ -9,9 +9,11 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
         upload(name)
 
         on(accessory.host) do
-          execute *MRSK.auditor.record("accessory #{name} boot"), verbosity: :debug
+          execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug
           execute *accessory.run
         end
+
+        audit_broadcast "Booted accessory #{name}"
       end
     end
   end
@@ -20,8 +22,6 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   def upload(name)
     with_accessory(name) do |accessory|
       on(accessory.host) do
-        execute *MRSK.auditor.record("accessory #{name} upload files"), verbosity: :debug
-
         accessory.files.each do |(local, remote)|
           accessory.ensure_local_file_present(local)
 
@@ -37,8 +37,6 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   def directories(name)
     with_accessory(name) do |accessory|
       on(accessory.host) do
-        execute *MRSK.auditor.record("accessory #{name} create directories"), verbosity: :debug
-
         accessory.directories.keys.each do |host_path|
           execute *accessory.make_directory(host_path)
         end
@@ -59,7 +57,7 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   def start(name)
     with_accessory(name) do |accessory|
       on(accessory.host) do
-        execute *MRSK.auditor.record("accessory #{name} start"), verbosity: :debug
+        execute *MRSK.auditor.record("Started #{name} accessory"), verbosity: :debug
         execute *accessory.start
       end
     end
@@ -69,7 +67,7 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   def stop(name)
     with_accessory(name) do |accessory|
       on(accessory.host) do
-        execute *MRSK.auditor.record("accessory #{name} stop"), verbosity: :debug
+        execute *MRSK.auditor.record("Stopped #{name} accessory"), verbosity: :debug
         execute *accessory.stop, raise_on_non_zero_exit: false
       end
     end
@@ -111,14 +109,14 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
       when options[:reuse]
         say "Launching command from existing container...", :magenta
         on(accessory.host) do
-          execute *MRSK.auditor.record("accessory #{name} cmd '#{cmd}'"), verbosity: :debug
+          execute *MRSK.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
           capture_with_info(*accessory.execute_in_existing_container(cmd))
         end
 
       else
         say "Launching command from new container...", :magenta
         on(accessory.host) do
-          execute *MRSK.auditor.record("accessory #{name} cmd '#{cmd}'"), verbosity: :debug
+          execute *MRSK.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
           capture_with_info(*accessory.execute_in_new_container(cmd))
         end
       end
@@ -169,7 +167,7 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   def remove_container(name)
     with_accessory(name) do |accessory|
       on(accessory.host) do
-        execute *MRSK.auditor.record("accessory #{name} remove container"), verbosity: :debug
+        execute *MRSK.auditor.record("Remove #{name} accessory container"), verbosity: :debug
         execute *accessory.remove_container
       end
     end
@@ -179,7 +177,7 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   def remove_image(name)
     with_accessory(name) do |accessory|
       on(accessory.host) do
-        execute *MRSK.auditor.record("accessory #{name} remove image"), verbosity: :debug
+        execute *MRSK.auditor.record("Removed #{name} accessory image"), verbosity: :debug
         execute *accessory.remove_image
       end
     end
@@ -189,7 +187,6 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   def remove_service_directory(name)
     with_accessory(name) do |accessory|
       on(accessory.host) do
-        execute *MRSK.auditor.record("accessory #{name} remove service directory"), verbosity: :debug
         execute *accessory.remove_service_directory
       end
     end

lib/mrsk/cli/app.rb

@@ -7,7 +7,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
 
       MRSK.config.roles.each do |role|
         on(role.hosts) do |host|
-          execute *MRSK.auditor.record("app boot version #{version}"), verbosity: :debug
+          execute *MRSK.auditor.record("Booted app version #{version}"), verbosity: :debug
 
           begin
             execute *MRSK.app.stop, raise_on_non_zero_exit: false
@@ -15,7 +15,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
           rescue SSHKit::Command::Failed => e
             if e.message =~ /already in use/
               error "Rebooting container with same version already deployed on #{host}"
-              execute *MRSK.auditor.record("app rebooted with version #{version}"), verbosity: :debug
+              execute *MRSK.auditor.record("Rebooted app version #{version}"), verbosity: :debug
 
               execute *MRSK.app.remove_container(version: version)
               execute *MRSK.app.run(role: role.name)
@@ -31,7 +31,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   desc "start", "Start existing app on servers (use --version=<git-hash> to designate specific version)"
   def start
     on(MRSK.hosts) do
-      execute *MRSK.auditor.record("app start version #{MRSK.version}"), verbosity: :debug
+      execute *MRSK.auditor.record("Started app version #{MRSK.version}"), verbosity: :debug
       execute *MRSK.app.start, raise_on_non_zero_exit: false
     end
   end
@@ -39,7 +39,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   desc "stop", "Stop app on servers"
   def stop
     on(MRSK.hosts) do
-      execute *MRSK.auditor.record("app stop"), verbosity: :debug
+      execute *MRSK.auditor.record("Stopped app"), verbosity: :debug
       execute *MRSK.app.stop, raise_on_non_zero_exit: false
     end
   end
@@ -74,7 +74,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
         say "Launching command with version #{version} from existing container...", :magenta
 
         on(MRSK.hosts) do |host|
-          execute *MRSK.auditor.record("app cmd '#{cmd}' with version #{version}"), verbosity: :debug
+          execute *MRSK.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
           puts_by_host host, capture_with_info(*MRSK.app.execute_in_existing_container(cmd))
         end
       end
@@ -84,7 +84,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
       using_version(options[:version] || most_recent_version_available) do |version|
         say "Launching command with version #{version} from new container...", :magenta
         on(MRSK.hosts) do |host|
-          execute *MRSK.auditor.record("app cmd '#{cmd}' with version #{version}"), verbosity: :debug
+          execute *MRSK.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
           puts_by_host host, capture_with_info(*MRSK.app.execute_in_new_container(cmd))
         end
       end
@@ -140,7 +140,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   desc "remove_container [VERSION]", "Remove app container with given version from servers"
   def remove_container(version)
     on(MRSK.hosts) do
-      execute *MRSK.auditor.record("app remove container #{version}"), verbosity: :debug
+      execute *MRSK.auditor.record("Removed app container with version #{version}"), verbosity: :debug
       execute *MRSK.app.remove_container(version: version)
     end
   end
@@ -148,7 +148,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   desc "remove_containers", "Remove all app containers from servers"
   def remove_containers
     on(MRSK.hosts) do
-      execute *MRSK.auditor.record("app remove containers"), verbosity: :debug
+      execute *MRSK.auditor.record("Removed all app containers"), verbosity: :debug
       execute *MRSK.app.remove_containers
     end
   end
@@ -156,7 +156,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   desc "remove_images", "Remove all app images from servers"
   def remove_images
     on(MRSK.hosts) do
-      execute *MRSK.auditor.record("app remove images"), verbosity: :debug
+      execute *MRSK.auditor.record("Removed all app images"), verbosity: :debug
       execute *MRSK.app.remove_images
     end
   end

lib/mrsk/cli/base.rb

@@ -59,9 +59,14 @@ module Mrsk::Cli
       def print_runtime
         started_at = Time.now
         yield
+        return Time.now - started_at
       ensure
         runtime = Time.now - started_at
         puts "  Finished all in #{sprintf("%.1f seconds", runtime)}"
       end
+
+      def audit_broadcast(line)
+        run_locally { execute *MRSK.auditor.broadcast(line), verbosity: :debug }
+      end
   end
 end

lib/mrsk/cli/build.rb

@@ -29,7 +29,7 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
   desc "pull", "Pull app image from the registry onto servers"
   def pull
     on(MRSK.hosts) do
-      execute *MRSK.auditor.record("build pull image #{MRSK.version}"), verbosity: :debug
+      execute *MRSK.auditor.record("Pulled image with version #{MRSK.version}"), verbosity: :debug
       execute *MRSK.builder.pull
     end
   end

lib/mrsk/cli/healthcheck.rb

@@ -0,0 +1,29 @@
+class Mrsk::Cli::Healthcheck < Mrsk::Cli::Base
+  desc "perform", "Health check the current version of the app"
+  def perform
+    on(MRSK.primary_host) do
+      begin
+        execute *MRSK.healthcheck.run
+
+        target = "Health check against #{MRSK.config.healthcheck["path"]}"
+
+        if capture_with_info(*MRSK.healthcheck.curl) == "200"
+          info "#{target} succeeded with 200 OK!"
+        else
+          # Catches 1xx, 2xx, 3xx
+          raise SSHKit::Command::Failed, "#{target} failed to return 200 OK!"
+        end
+      rescue SSHKit::Command::Failed => e
+        if e.message =~ /curl/
+          # Catches 4xx, 5xx
+          raise SSHKit::Command::Failed, "#{target} failed to return 200 OK!"
+        else
+          raise
+        end
+      ensure
+        execute *MRSK.healthcheck.stop, raise_on_non_zero_exit: false
+        execute *MRSK.healthcheck.remove, raise_on_non_zero_exit: false
+      end
+    end
+  end
+end

lib/mrsk/cli/main.rb

@@ -10,7 +10,7 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
 
   desc "deploy", "Deploy the app to servers"
   def deploy
-    print_runtime do
+    runtime = print_runtime do
       say "Ensure Docker is installed...", :magenta
       invoke "mrsk:cli:server:bootstrap"
 
@@ -23,33 +23,48 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       say "Ensure Traefik is running...", :magenta
       invoke "mrsk:cli:traefik:boot"
 
+      say "Ensure app can pass healthcheck...", :magenta
+      invoke "mrsk:cli:healthcheck:perform"
+
       invoke "mrsk:cli:app:boot"
 
       say "Prune old containers and images...", :magenta
       invoke "mrsk:cli:prune:all"
     end
+
+    audit_broadcast "Deployed app in #{runtime.to_i} seconds"
   end
 
   desc "redeploy", "Deploy new version of the app to servers (without bootstrapping servers, starting Traefik, pruning, and registry login)"
   def redeploy
-    print_runtime do
+    runtime = print_runtime do
       say "Build and push app image...", :magenta
       invoke "mrsk:cli:build:deliver"
 
+      say "Ensure app can pass healthcheck...", :magenta
+      invoke "mrsk:cli:healthcheck:perform"
+
       invoke "mrsk:cli:app:boot"
     end
+
+    audit_broadcast "Redeployed app in #{runtime.to_i} seconds"
   end
 
   desc "rollback [VERSION]", "Rollback the app to VERSION"
   def rollback(version)
     MRSK.version = version
 
-    cli = self
+    if container_name_available?(MRSK.config.service_with_version)
+      say "Stop current version, then start version #{version}...", :magenta
 
-    cli.say "Stop current version, then start version #{version}...", :magenta
+      on(MRSK.hosts) do |host|
-    on(MRSK.hosts) do
+        execute *MRSK.app.stop, raise_on_non_zero_exit: false
-      execute *MRSK.app.stop, raise_on_non_zero_exit: false
+        execute *MRSK.app.start
-      execute *MRSK.app.start
+      end
+
+      audit_broadcast "Rolled back app to version #{version}"
+    else
+      say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
     end
   end
 
@@ -138,6 +153,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "build", "Build the application image"
   subcommand "build", Mrsk::Cli::Build
 
+  desc "healthcheck", "Healthcheck the application"
+  subcommand "healthcheck", Mrsk::Cli::Healthcheck
+
   desc "prune", "Prune old application images and containers"
   subcommand "prune", Mrsk::Cli::Prune
 
@@ -149,4 +167,11 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
 
   desc "traefik", "Manage the Traefik load balancer"
   subcommand "traefik", Mrsk::Cli::Traefik
+
+  private
+    def container_name_available?(container_name, host: MRSK.primary_host)
+      container_names = nil
+      on(host) { container_names = capture_with_info(*MRSK.app.list_container_names).split("\n") }
+      Array(container_names).include?(container_name)
+    end
 end

lib/mrsk/cli/prune.rb

@@ -5,10 +5,10 @@ class Mrsk::Cli::Prune < Mrsk::Cli::Base
     invoke :images
   end
 
-  desc "images", "Prune unused images older than 30 days"
+  desc "images", "Prune unused images older than 7 days"
   def images
     on(MRSK.hosts) do
-      execute *MRSK.auditor.record("prune images"), verbosity: :debug
+      execute *MRSK.auditor.record("Pruned images"), verbosity: :debug
       execute *MRSK.prune.images
     end
   end
@@ -16,7 +16,7 @@ class Mrsk::Cli::Prune < Mrsk::Cli::Base
   desc "containers", "Prune stopped containers for the service older than 3 days"
   def containers
     on(MRSK.hosts) do
-      execute *MRSK.auditor.record("prune containers"), verbosity: :debug
+      execute *MRSK.auditor.record("Pruned containers"), verbosity: :debug
       execute *MRSK.prune.containers
     end
   end

lib/mrsk/cli/traefik.rb

@@ -14,7 +14,7 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
   desc "start", "Start existing Traefik on servers"
   def start
     on(MRSK.traefik_hosts) do
-      execute *MRSK.auditor.record("traefik start"), verbosity: :debug
+      execute *MRSK.auditor.record("Started traefik"), verbosity: :debug
       execute *MRSK.traefik.start, raise_on_non_zero_exit: false
     end
   end
@@ -22,7 +22,7 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
   desc "stop", "Stop Traefik on servers"
   def stop
     on(MRSK.traefik_hosts) do
-      execute *MRSK.auditor.record("traefik stop"), verbosity: :debug
+      execute *MRSK.auditor.record("Stopped traefik"), verbosity: :debug
       execute *MRSK.traefik.stop, raise_on_non_zero_exit: false
     end
   end
@@ -72,7 +72,7 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
   desc "remove_container", "Remove Traefik container from servers"
   def remove_container
     on(MRSK.traefik_hosts) do
-      execute *MRSK.auditor.record("traefik remove container"), verbosity: :debug
+      execute *MRSK.auditor.record("Removed traefik container"), verbosity: :debug
       execute *MRSK.traefik.remove_container
     end
   end
@@ -80,7 +80,7 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
   desc "remove_container", "Remove Traefik image from servers"
   def remove_image
     on(MRSK.traefik_hosts) do
-      execute *MRSK.auditor.record("traefik remove image"), verbosity: :debug
+      execute *MRSK.auditor.record("Removed traefik image"), verbosity: :debug
       execute *MRSK.traefik.remove_image
     end
   end

lib/mrsk/commander.rb

@@ -73,6 +73,10 @@ class Mrsk::Commander
     @auditor ||= Mrsk::Commands::Auditor.new(config)
   end
 
+  def healthcheck
+    @healthcheck ||= Mrsk::Commands::Healthcheck.new(config)
+  end
+
 
   def with_verbosity(level)
     old_level = self.verbosity

lib/mrsk/commands/app.rb

@@ -75,10 +75,6 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
     docker :ps, "-q", *service_filter
   end
 
-  def container_id_for(container_name:)
-    docker :container, :ls, "-a", "-f", "name=#{container_name}", "-q"
-  end
-
   def current_running_version
     # FIXME: Find more graceful way to extract the version from "app-version" than using sed and tail!
     pipe \
@@ -93,11 +89,21 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
       "head -n 1"
   end
 
+  def all_versions_from_available_containers
+    pipe \
+      docker(:image, :ls, "--format", '"{{.Tag}}"', config.repository),
+      "head -n 1"
+  end
+
 
   def list_containers
     docker :container, :ls, "-a", *service_filter
   end
 
+  def list_container_names
+    [ *list_containers, "--format", "'{{ .Names }}'" ]
+  end
+
   def remove_container(version:)
     pipe \
       container_id_for(container_name: service_with_version(version)),

lib/mrsk/commands/auditor.rb

@@ -1,12 +1,22 @@
 require "active_support/core_ext/time/conversions"
 
 class Mrsk::Commands::Auditor < Mrsk::Commands::Base
+  # Runs remotely
   def record(line)
     append \
       [ :echo, tagged_line(line) ],
       audit_log_file
   end
 
+  # Runs locally
+  def broadcast(line)
+    if broadcast_cmd = config.audit_broadcast_cmd
+      pipe \
+        [ :echo, tagged_line(line) ],
+        broadcast_cmd
+    end
+  end
+
   def reveal
     [ :tail, "-n", 50, audit_log_file ]
   end
@@ -21,14 +31,14 @@ class Mrsk::Commands::Auditor < Mrsk::Commands::Base
     end
 
     def tags
-      "[#{timestamp}] [#{performer}]"
+      "[#{recorded_at}] [#{performer}]"
     end
 
     def performer
-      `whoami`.strip
+      @performer ||= `whoami`.strip
     end
 
-    def timestamp
+    def recorded_at
       Time.now.to_fs(:db)
     end
 end

lib/mrsk/commands/base.rb

@@ -17,6 +17,10 @@ module Mrsk::Commands
       end
     end
 
+    def container_id_for(container_name:)
+      docker :container, :ls, "-a", "-f", "name=#{container_name}", "-q"
+    end
+
     private
       def combine(*commands, by: "&&")
         commands

lib/mrsk/commands/healthcheck.rb

@@ -0,0 +1,46 @@
+class Mrsk::Commands::Healthcheck < Mrsk::Commands::Base
+  EXPOSED_PORT = 3999
+
+  def run
+    web = config.role(:web)
+
+    docker :run,
+      "-d",
+      "--name", container_name_with_version,
+      "-p", "#{EXPOSED_PORT}:#{config.healthcheck["port"]}",
+      "--label", "service=#{container_name}",
+      *web.env_args,
+      *config.volume_args,
+      config.absolute_image,
+      web.cmd
+  end
+
+  def curl
+    [ :curl, "--silent", "--output", "/dev/null", "--write-out", "'%{http_code}'", health_url ]
+  end
+
+  def stop
+    pipe \
+      container_id_for(container_name: container_name),
+      xargs(docker(:stop))
+  end
+
+  def remove
+    pipe \
+      container_id_for(container_name: container_name),
+      xargs(docker(:container, :rm))
+  end
+
+  private
+    def container_name
+      "healthcheck-#{config.service}"
+    end
+
+    def container_name_with_version
+      "healthcheck-#{config.service_with_version}"
+    end
+
+    def health_url
+      "http://localhost:#{EXPOSED_PORT}#{config.healthcheck["path"]}"
+    end
+end

lib/mrsk/commands/prune.rb

@@ -2,14 +2,11 @@ require "active_support/duration"
 require "active_support/core_ext/numeric/time"
 
 class Mrsk::Commands::Prune < Mrsk::Commands::Base
-  PRUNE_IMAGES_AFTER     = 7.days.in_hours.to_i
+  def images(until_hours: 7.days.in_hours.to_i)
-  PRUNE_CONTAINERS_AFTER = 3.days.in_hours.to_i
+    docker :image, :prune, "--all", "--force", "--filter", "label=service=#{config.service}", "--filter", "until=#{until_hours}h"
-
-  def images
-    docker :image, :prune, "--all", "--force", "--filter", "label=service=#{config.service}", "--filter", "until=#{PRUNE_IMAGES_AFTER}h"
   end
 
-  def containers
+  def containers(until_hours: 3.days.in_hours.to_i)
-    docker :container, :prune, "--force", "--filter", "label=service=#{config.service}", "--filter", "until=#{PRUNE_CONTAINERS_AFTER}h"
+    docker :container, :prune, "--force", "--filter", "label=service=#{config.service}", "--filter", "until=#{until_hours}h"
   end
 end

lib/mrsk/configuration.rb

@@ -107,6 +107,7 @@ class Mrsk::Configuration
     end
   end
 
+
   def ssh_user
     if raw_config.ssh.present?
       raw_config.ssh["user"] || "root"
@@ -127,6 +128,15 @@ class Mrsk::Configuration
   end
 
 
+  def audit_broadcast_cmd
+    raw_config.audit_broadcast_cmd
+  end
+
+  def healthcheck
+    { "path" => "/up", "port" => 3000 }.merge(raw_config.healthcheck || {})
+  end
+
+
   def valid?
     ensure_required_keys_present && ensure_env_available
   end
@@ -145,7 +155,8 @@ class Mrsk::Configuration
       volume_args: volume_args,
       ssh_options: ssh_options,
       builder: raw_config.builder,
-      accessories: raw_config.accessories
+      accessories: raw_config.accessories,
+      healthcheck: healthcheck
     }.compact
   end
 

lib/mrsk/configuration/role.rb

@@ -59,7 +59,7 @@ class Mrsk::Configuration::Role
       if running_traefik?
         {
           "traefik.http.routers.#{config.service}.rule" => "'PathPrefix(`/`)'",
-          "traefik.http.services.#{config.service}.loadbalancer.healthcheck.path" => "/up",
+          "traefik.http.services.#{config.service}.loadbalancer.healthcheck.path" => config.healthcheck["path"],
           "traefik.http.services.#{config.service}.loadbalancer.healthcheck.interval" => "1s",
           "traefik.http.middlewares.#{config.service}.retry.attempts" => "3",
           "traefik.http.middlewares.#{config.service}.retry.initialinterval" => "500ms"

lib/mrsk/version.rb

@@ -1,3 +1,3 @@
 module Mrsk
-  VERSION = "0.6.4"
+  VERSION = "0.7.0"
 end

test/cli/main_test.rb

@@ -5,4 +5,29 @@ class CliMainTest < CliTestCase
     version = stdouted { Mrsk::Cli::Main.new.version }
     assert_equal Mrsk::VERSION, version
   end
+
+  test "rollback bad version" do
+    run_command("details") # Preheat MRSK const
+
+    run_command("rollback", "nonsense").tap do |output|
+      assert_match /docker container ls -a --filter label=service=app --format '{{ .Names }}'/, output
+      assert_match /The app version 'nonsense' is not available as a container/, output
+    end
+  end
+
+  test "rollback good version" do
+    Mrsk::Cli::Main.any_instance.stubs(:container_name_available?).returns(true)
+
+    run_command("rollback", "123").tap do |output|
+      assert_match /Stop current version, then start version 123/, output
+      assert_match /docker ps -q --filter label=service=app | xargs docker stop/, output
+      assert_match /docker start app-123/, output
+    end
+  end
+
+
+  private
+    def run_command(*command)
+      stdouted { Mrsk::Cli::Main.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+    end
 end

test/commands/app_test.rb

@@ -26,6 +26,14 @@ class CommandsAppTest < ActiveSupport::TestCase
       @app.run.join(" ")
   end
 
+  test "run with custom healthcheck path" do
+    @config[:healthcheck] = { "path" => "/healthz" }
+
+    assert_equal \
+      "docker run -d --restart unless-stopped --log-opt max-size=10m --name app-999 -e RAILS_MASTER_KEY=456 --label service=app --label role=web --label traefik.http.routers.app.rule='PathPrefix(`/`)' --label traefik.http.services.app.loadbalancer.healthcheck.path=/healthz --label traefik.http.services.app.loadbalancer.healthcheck.interval=1s --label traefik.http.middlewares.app.retry.attempts=3 --label traefik.http.middlewares.app.retry.initialinterval=500ms dhh/app:999",
+      @app.run.join(" ")
+  end
+
   test "start" do
     assert_equal \
       "docker start app-999",

test/commands/auditor_test.rb

@@ -0,0 +1,27 @@
+require "test_helper"
+
+class CommandsAuditorTest < ActiveSupport::TestCase
+  setup do
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      audit_broadcast_cmd: "bin/audit_broadcast"
+    }
+  end
+
+  test "record" do
+    assert_match \
+      /echo '.* app removed container' >> mrsk-app-audit.log/,
+      new_command.record("app removed container").join(" ")
+  end
+
+  test "broadcast" do
+    assert_match \
+      /echo '.* app removed container' \| bin\/audit_broadcast/,
+      new_command.broadcast("app removed container").join(" ")
+  end
+
+  private
+    def new_command
+      Mrsk::Commands::Auditor.new(Mrsk::Configuration.new(@config, version: "123"))
+    end
+end

test/commands/healthcheck_test.rb

@@ -0,0 +1,55 @@
+require "test_helper"
+
+class CommandsHealthcheckTest < ActiveSupport::TestCase
+  setup do
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
+    }
+  end
+
+  test "run" do
+    assert_equal \
+      "docker run -d --name healthcheck-app-123 -p 3999:3000 --label service=healthcheck-app dhh/app:123",
+      new_command.run.join(" ")
+  end
+
+  test "run with custom port" do
+    @config[:healthcheck] = { "port" => 3001 }
+
+    assert_equal \
+      "docker run -d --name healthcheck-app-123 -p 3999:3001 --label service=healthcheck-app dhh/app:123",
+      new_command.run.join(" ")
+  end
+
+  test "curl" do
+    assert_equal \
+      "curl --silent --output /dev/null --write-out '%{http_code}' http://localhost:3999/up",
+      new_command.curl.join(" ")
+  end
+
+  test "curl with custom path" do
+    @config[:healthcheck] = { "path" => "/healthz" }
+
+    assert_equal \
+      "curl --silent --output /dev/null --write-out '%{http_code}' http://localhost:3999/healthz",
+      new_command.curl.join(" ")
+  end
+
+  test "stop" do
+    assert_equal \
+      "docker container ls -a -f name=healthcheck-app -q | xargs docker stop",
+      new_command.stop.join(" ")
+  end
+
+  test "remove" do
+    assert_equal \
+      "docker container ls -a -f name=healthcheck-app -q | xargs docker container rm",
+      new_command.remove.join(" ")
+  end
+
+  private
+    def new_command
+      Mrsk::Commands::Healthcheck.new(Mrsk::Configuration.new(@config, version: "123"))
+    end
+end