Bump version for 0.12.1

Merge pull request #223 from basecamp/customizable-audit-broadcast
Allow customizing audit broadcast with env
2023-05-05 14:33:25 +02:00 · 2023-05-05 14:30:04 +02:00 · 2023-05-05 14:28:22 +02:00 · 2023-05-05 12:13:14 +01:00 · 2023-05-05 11:40:29 +01:00 · 2023-05-05 11:25:22 +02:00
76 changed files with 1641 additions and 369 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,5 +1,9 @@
 name: CI
-on: [push, pull_request]
+on: 
+  push:
+    branches:
+      - main
+  pull_request:
 jobs:
  tests:
    strategy:
--- a/6
+++ b/6
@@ -14,7 +14,7 @@ COPY Gemfile Gemfile.lock mrsk.gemspec ./
 COPY lib/mrsk/version.rb /mrsk/lib/mrsk/version.rb

 # Install system dependencies
-RUN apk add --no-cache --update build-base git docker openrc \
+RUN apk add --no-cache --update build-base git docker openrc openssh-client-default \
    && rc-update add docker boot \
    && gem install bundler --version=2.4.3 \
    && bundle install
@@ -31,6 +31,10 @@ RUN gem build mrsk.gemspec && \
 # Set the working directory to /workdir
 WORKDIR /workdir

+# Tell git it's safe to access /workdir/.git even if
+# the directory is owned by a different user
+RUN git config --global --add safe.directory /workdir
+
 # Set the entrypoint to run the installed binary in /workdir
 # Example:  docker run -it -v "$PWD:/workdir" mrsk init
 ENTRYPOINT ["mrsk"]
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,11 +1,12 @@
 PATH
  remote: .
  specs:
-    mrsk (0.10.1)
+    mrsk (0.12.1)
      activesupport (>= 7.0)
      bcrypt_pbkdf (~> 1.0)
      dotenv (~> 2.8)
      ed25519 (~> 1.2)
+      net-ssh (~> 7.0)
      sshkit (~> 1.21)
      thor (~> 1.2)
      zeitwerk (~> 2.5)
@@ -35,13 +36,18 @@ GEM
    builder (3.2.4)
    concurrent-ruby (1.2.2)
    crass (1.0.6)
-    debug (1.7.1)
+    debug (1.7.2)
+      irb (>= 1.5.0)
+      reline (>= 0.3.1)
    dotenv (2.8.1)
    ed25519 (1.3.0)
    erubi (1.12.0)
    i18n (1.12.0)
      concurrent-ruby (~> 1.0)
-    loofah (2.19.1)
+    io-console (0.6.0)
+    irb (1.6.3)
+      reline (>= 0.3.0)
+    loofah (2.20.0)
      crass (~> 1.0.2)
      nokogiri (>= 1.5.9)
    method_source (1.0.0)
@@ -74,6 +80,8 @@ GEM
      thor (~> 1.0)
      zeitwerk (~> 2.5)
    rake (13.0.6)
+    reline (0.3.3)
+      io-console (~> 0.5)
    ruby2_keywords (0.0.5)
    sshkit (1.21.4)
      net-scp (>= 1.1.2)
--- a/README.md
+++ b/README.md
@@ -6,6 +6,8 @@ Watch the screencast: https://www.youtube.com/watch?v=LL1cV2FXZ5I

 Join us on Discord: https://discord.gg/YgHVT7GCXS

+Ask questions: https://github.com/mrsked/mrsk/discussions
+
 ## Installation

 If you have a Ruby environment available, you can install MRSK globally with:
@@ -14,13 +16,13 @@ If you have a Ruby environment available, you can install MRSK globally with:
 gem install mrsk
 ```

-...otherwise, you can run a dockerized version via an alias (add this to your ${SHELL}rc to simplify re-use):
+...otherwise, you can run a dockerized version via an alias (add this to your .bashrc or similar to simplify re-use):

 ```sh
 alias mrsk='docker run --rm -it -v $HOME/.ssh:/root/.ssh -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}/:/workdir  ghcr.io/mrsked/mrsk'
 ```

-Then, inside your app directory, run `mrsk init` (or `mrsk init --bundle` within Rails apps where you want a bin/mrsk binstub). Now edit the new file `config/deploy.yml`. It could look as simple as this:
+Then, inside your app directory, run `mrsk init` (or `mrsk init --bundle` within Rails 7+ apps where you want a bin/mrsk binstub). Now edit the new file `config/deploy.yml`. It could look as simple as this:

 ```yaml
 service: hey
@@ -191,6 +193,15 @@ ssh:
  user: app
 ```

+If you are using non-root user, you need to bootstrap your servers manually, before using them with MRSK. On Ubuntu, you'd do:
+
+```bash
+sudo apt update
+sudo apt upgrade -y
+sudo apt install -y docker.io curl git
+sudo usermod -a -G docker ubuntu
+```
+
 ### Using a proxy SSH host

 If you need to connect to server through a proxy host, you can use `ssh/proxy`:
@@ -207,6 +218,13 @@ ssh:
  proxy: "app@192.168.0.1"
 ```

+Also if you need specific proxy command to connect to the server:
+
+```yaml
+ssh:
+  proxy_command: aws ssm start-session --target %h --document-name AWS-StartSSHSession --parameters 'portNumber=%p' --region=us-east-1 ## ssh via aws ssm
+```
+
 ### Using env variables

 You can inject env variables into the app containers using `env`:
@@ -288,8 +306,9 @@ You can specialize the default Traefik rules by setting labels on the containers

 ```yaml
 labels:
-  traefik.http.routers.hey.rule: Host(`app.hey.com`)
+  traefik.http.routers.hey-web.rule: Host(`app.hey.com`)
 ```
+Traefik rules are in the "service-role-destination" format. The default role will be `web` if no rule is specified. If the destination is not specified, it is not included. To give an example, the above rule would become "traefik.http.routers.hey-web-staging.rule" if it was for the "staging" destination.

 Note: The backticks are needed to ensure the rule is passed in correctly and not treated as command substitution by Bash!

@@ -312,6 +331,21 @@ servers:
      my-label: "50"
 ```

+### Using shell expansion
+
+You can use shell expansion to interpolate values from the host machine into labels and env variables with the `${}` syntax.
+Anything within the curly braces will be executed on the host machine and the result will be interpolated into the label or env variable.
+
+```yaml
+labels:
+  host-machine: "${cat /etc/hostname}"
+
+env:
+  HOST_DEPLOYMENT_DIR: "${PWD}"
+```
+
+Note: Any other occurrence of `$` will be escaped to prevent unwanted shell expansion!
+
 ### Using container options

 You can specialize the options used to start containers using the `options` definitions:
@@ -439,9 +473,9 @@ RUN --mount=type=secret,id=GITHUB_TOKEN \
  rm -rf /usr/local/bundle/cache
 ```

-### Using command arguments for Traefik
+### Traefik command arguments

-You can customize the traefik command line:
+Customize the Traefik command line using `args`:

 ```yaml
 traefik:
@@ -450,20 +484,38 @@ traefik:
    accesslog.format: json
 ```

-This will start the traefik container with `--accesslog=true accesslog.format=json`.
+This starts the Traefik container with `--accesslog=true --accesslog.format=json` arguments.

-### Traefik's host port binding
+### Traefik host port binding

-By default Traefik binds to port 80 of the host machine, it can be configured to use an alternative port:
+Traefik binds to port 80 by default. Specify an alternative port using `host_port`:

 ```yaml
 traefik:
  host_port: 8080
 ```

-### Configure docker options for traefik
+### Traefik version, upgrades, and custom images

-We allow users to pass additional docker options to the trafik container like
+MRSK runs the traefik:v2.9 image to track Traefik 2.9.x releases.
+
+To pin Traefik to a specific version or an image published to your registry,
+specify `image`:
+
+```yaml
+traefik:
+  image: traefik:v2.10.0-rc1
+```
+
+This is useful for downgrading Traefik if there's an unexpected breaking
+change in a minor version release, upgrading Traefik to test forthcoming
+releases, or running your own Traefik-derived image.
+
+MRSK has not been tested for compatibility with Traefik 3 betas. Please do!
+
+### Traefik container configuration
+
+Pass additional Docker configuration for the Traefik container using `options`:

 ```yaml
 traefik:
@@ -475,12 +527,27 @@ traefik:
    memory: 512m
 ```

-This will start the traefik container with a command like: `docker run ... --volume /tmp/example.json:/tmp/example.json --publish 8080:8080 `
+This starts the Traefik container with `--volume /tmp/example.json:/tmp/example.json --publish 8080:8080 --memory 512m` arguments to `docker run`.

+### Traefik container labels

-### Configure alternate entrypoints for traefik
+Add labels to Traefik Docker container.

-You can configure multiple entrypoints for traefik like so:
+```yaml
+traefik:
+  labels:
+    traefik.enable: true
+    traefik.http.routers.dashboard.rule: Host(`traefik.example.com`) && (PathPrefix(`/api`) || PathPrefix(`/dashboard`))
+    traefik.http.routers.dashboard.service: api@internal
+    traefik.http.routers.dashboard.middlewares: auth
+    traefik.http.middlewares.auth.basicauth.users: test:$2y$05$H2o72tMaO.TwY1wNQUV1K.fhjRgLHRDWohFvUZOJHBEtUXNKrqUKi # test:password
+```
+
+This labels Traefik container with `--label traefik.http.routers.dashboard.middlewares=\"auth\"` and so on.
+
+### Traefik alternate entrypoints
+
+You can configure multiple entrypoints for Traefik like so:

 ```yaml
 service: myservice
@@ -540,7 +607,7 @@ accessories:
      memory: "2GB"
  redis:
    image: redis:latest
-    role:
+    roles:
      - web
    port: "36379:6379"
    volumes:
@@ -610,18 +677,60 @@ That'll post a line like follows to a preconfigured chatbot in Basecamp:
 [My App] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de
 ```

-### Using custom healthcheck path or port
+`MRSK_*` environment variables are available to the broadcast command for
+fine-grained audit reporting, e.g. for triggering deployment reports or
+firing a JSON webhook. These variables include:
+- `MRSK_RECORDED_AT` - UTC timestamp in ISO 8601 format, e.g. `2023-04-14T17:07:31Z`
+- `MRSK_PERFORMER` - the local user performing the command (from `whoami`)
+- `MRSK_MESSAGE` - the full audit message, e.g. "Deployed app@150b24f"
+- `MRSK_DESTINATION` - optional: destination, e.g. "staging"
+- `MRSK_ROLE` - optional: role targeted, e.g. "web"

-MRSK defaults to checking the health of your application again `/up` on port 3000. You can tailor both with the `healthcheck` setting:
+Use `mrsk broadcast` to test and troubleshoot your broadcast command:
+
+```bash
+mrsk broadcast -m "test audit message"
+```
+
+### Healthcheck
+
+MRSK uses Docker healtchecks to check the health of your application during deployment. Traefik uses this same healthcheck status to determine when a container is ready to receive traffic.
+
+The healthcheck defaults to testing the HTTP response to the path `/up` on port 3000, up to 7 times. You can tailor this behaviour with the `healthcheck` setting:

 ```yaml
 healthcheck:
  path: /healthz
  port: 4000
+  max_attempts: 7
 ```

 This will ensure your application is configured with a traefik label for the healthcheck against `/healthz` and that the pre-deploy healthcheck that MRSK performs is done against the same path on port 4000.

+You can also specify a custom healthcheck command, which is useful for non-HTTP services:
+
+```yaml
+healthcheck:
+  cmd: /bin/check_health
+```
+
+The top-level healthcheck configuration applies to all services that use
+Traefik, by default. You can also specialize the configuration at the role
+level:
+
+```yaml
+servers:
+  job:
+    hosts: ...
+    cmd: bin/jobs
+    healthcheck:
+      cmd: bin/check
+```
+
+The healthcheck allows for an optional `max_attempts` setting, which will attempt the healthcheck up to the specified number of times before failing the deploy. This is useful for applications that take a while to start up. The default is 7.
+
+Note: The HTTP health checks assume that the `curl` command is available inside the container. If that's not the case, use the healthcheck's `cmd` option to specify an alternative check that the container supports.
+
 ## Commands

 ### Running commands on servers
@@ -761,6 +870,24 @@ mrsk lock acquire -m "Doing maintanence"
 mrsk lock release
 ```

+## Rolling deployments
+
+When deploying to large numbers of hosts, you might prefer not to restart your services on every host at the same time.
+
+MRSK's default is to boot new containers on all hosts in parallel. But you can control this by configuring `boot/limit` and `boot/wait` as options:
+
+```yaml
+service: myservice
+
+boot:
+  limit: 10 # Can also specify as a percentage of total hosts, such as "25%"
+  wait: 2
+```
+
+When `limit` is specified, containers will be booted on, at most, `limit` hosts at once. MRSK will pause for `wait` seconds between batches.
+
+These settings only apply when booting containers (using `mrsk deploy`, or `mrsk app boot`). For other commands, MRSK continues to run commands in parallel across all hosts.
+
 ## Stage of development

 This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com).
--- a/lib/mrsk/cli.rb
+++ b/lib/mrsk/cli.rb
@@ -1,4 +1,5 @@
 module Mrsk::Cli
+  class LockError < StandardError; end
 end

 # SSHKit uses instance eval, so we need a global const for ergonomics
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -6,27 +6,33 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
      using_version(version_or_latest) do |version|
        say "Start container with version #{version} using a #{MRSK.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta

-        cli = self
+        on(MRSK.hosts) do
+          execute *MRSK.auditor.record("Tagging #{MRSK.config.absolute_image} as the latest image"), verbosity: :debug
+          execute *MRSK.app.tag_current_as_latest
+        end

-        on(MRSK.hosts) do |host|
+        on(MRSK.hosts, **MRSK.boot_strategy) do |host|
          roles = MRSK.roles_on(host)

          roles.each do |role|
-            execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug
+            app = MRSK.app(role: role)
+            auditor = MRSK.auditor(role: role)

-            begin
-              if capture_with_info(*MRSK.app(role: role).container_id_for_version(version)).present?
+            execute *auditor.record("Booted app version #{version}"), verbosity: :debug
+
+            if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
              tmp_version = "#{version}_#{SecureRandom.hex(8)}"
              info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
-                execute *MRSK.auditor(role: role).record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
-                execute *MRSK.app(role: role).rename_container(version: version, new_version: tmp_version)
+              execute *auditor.record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
+              execute *app.rename_container(version: version, new_version: tmp_version)
            end

-              old_version = capture_with_info(*MRSK.app(role: role).current_running_version).strip
-              execute *MRSK.app(role: role).run
-              sleep MRSK.config.readiness_delay
-              execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
-            end
+            old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+            execute *app.run
+
+            Mrsk::Utils::HealthcheckPoller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+
+            execute *app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
          end
        end
      end
@@ -54,7 +60,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
        roles = MRSK.roles_on(host)

        roles.each do |role|
-          execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug
+          execute *MRSK.auditor.record("Stopped app", role: role), verbosity: :debug
          execute *MRSK.app(role: role).stop, raise_on_non_zero_exit: false
        end
      end
@@ -101,7 +107,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
          roles = MRSK.roles_on(host)

          roles.each do |role|
-            execute *MRSK.auditor(role: role).record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
+            execute *MRSK.auditor.record("Executed cmd '#{cmd}' on app version #{version}", role: role), verbosity: :debug
            puts_by_host host, capture_with_info(*MRSK.app(role: role).execute_in_existing_container(cmd))
          end
        end
@@ -124,6 +130,31 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
    on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_containers) }
  end

+  desc "stale_containers", "Detect app stale containers"
+  option :stop, aliases: "-s", type: :boolean, default: false, desc: "Stop the stale containers found"
+  def stale_containers
+    with_lock do
+      stop = options[:stop]
+
+      cli = self
+
+      on(MRSK.hosts) do |host|
+        roles = MRSK.roles_on(host)
+
+        roles.each do |role|
+          cli.send(:stale_versions, host: host, role: role).each do |version|
+            if stop
+              puts_by_host host, "Stopping stale container for role #{role} with version #{version}"
+              execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false
+            else
+              puts_by_host host,  "Detected stale container for role #{role} with version #{version} (use `mrsk app stale_containers --stop` to stop)"
+            end
+          end
+        end
+      end
+    end
+  end
+
  desc "images", "Show app images on servers"
  def images
    on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_images) }
@@ -183,7 +214,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
        roles = MRSK.roles_on(host)

        roles.each do |role|
-          execute *MRSK.auditor(role: role).record("Removed app container with version #{version}"), verbosity: :debug
+          execute *MRSK.auditor.record("Removed app container with version #{version}", role: role), verbosity: :debug
          execute *MRSK.app(role: role).remove_container(version: version)
        end
      end
@@ -197,7 +228,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
        roles = MRSK.roles_on(host)

        roles.each do |role|
-          execute *MRSK.auditor(role: role).record("Removed all app containers"), verbosity: :debug
+          execute *MRSK.auditor.record("Removed all app containers", role: role), verbosity: :debug
          execute *MRSK.app(role: role).remove_containers
        end
      end
@@ -240,6 +271,17 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
      version.presence
    end

+    def stale_versions(host:, role:)
+      versions = nil
+      on(host) do
+        versions = \
+          capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false)
+          .split("\n")
+          .drop(1)
+      end
+      versions
+    end
+
    def version_or_latest
      options[:version] || "latest"
    end
--- a/lib/mrsk/cli/base.rb
+++ b/lib/mrsk/cli/base.rb
@@ -6,8 +6,6 @@ module Mrsk::Cli
  class Base < Thor
    include SSHKit::DSL

-    class LockError < StandardError; end
-
    def self.exit_on_failure?() true end

    class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging"
@@ -79,32 +77,55 @@ module Mrsk::Cli
      end

      def with_lock
+        if MRSK.holding_lock?
+          yield
+        else
          acquire_lock

+          begin
            yield
-      ensure
+          rescue
+            if MRSK.hold_lock_on_error?
+              error "  \e[31mDeploy lock was not released\e[0m"
+            else
              release_lock
            end

-      def acquire_lock
-        if MRSK.lock_count == 0
-          say "Acquiring the deploy lock"
-          on(MRSK.primary_host) { execute *MRSK.lock.acquire("Automatic deploy lock", MRSK.config.version) }
-        end
-        MRSK.lock_count += 1
-      rescue SSHKit::Runner::ExecuteError => e
-        if e.message =~ /cannot create directory/
-          invoke "mrsk:cli:lock:status", []
+            raise
          end

+          release_lock
+        end
+      end
+
+      def acquire_lock
+        say "Acquiring the deploy lock"
+        on(MRSK.primary_host) { execute *MRSK.lock.acquire("Automatic deploy lock", MRSK.config.version) }
+
+        MRSK.holding_lock = true
+      rescue SSHKit::Runner::ExecuteError => e
+        if e.message =~ /cannot create directory/
+          on(MRSK.primary_host) { execute *MRSK.lock.status }
          raise LockError, "Deploy lock found"
+        else
+          raise e
+        end
      end

      def release_lock
-        MRSK.lock_count -= 1
-        if MRSK.lock_count == 0
        say "Releasing the deploy lock"
        on(MRSK.primary_host) { execute *MRSK.lock.release }
+
+        MRSK.holding_lock = false
+      end
+
+      def hold_lock_on_error
+        if MRSK.hold_lock_on_error?
+          yield
+        else
+          MRSK.hold_lock_on_error = true
+          yield
+          MRSK.hold_lock_on_error = false
        end
      end
  end
--- a/lib/mrsk/cli/build.rb
+++ b/lib/mrsk/cli/build.rb
@@ -1,4 +1,6 @@
 class Mrsk::Cli::Build < Mrsk::Cli::Base
+  class BuildError < StandardError; end
+
  desc "deliver", "Build app and push app image to registry then pull image on servers"
  def deliver
    with_lock do
@@ -14,7 +16,9 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base

      run_locally do
        begin
+          if cli.verify_local_dependencies
            MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
+          end
        rescue SSHKit::Command::Failed => e
          if e.message =~ /(no builder)|(no such file or directory)/
            error "Missing compatible builder, so creating a new one first"
@@ -77,4 +81,22 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
      puts capture(*MRSK.builder.info)
    end
  end
+
+
+  desc "", "" # Really a private method, but needed to be invoked from #push
+  def verify_local_dependencies
+    run_locally do
+      begin
+        execute *MRSK.builder.ensure_local_dependencies_installed
+      rescue SSHKit::Command::Failed => e
+        build_error = e.message =~ /command not found/ ?
+          "Docker is not installed locally" :
+          "Docker buildx plugin is not installed locally"
+
+        raise BuildError, build_error
+      end
+    end
+
+    true
+  end
 end
--- a/lib/mrsk/cli/healthcheck.rb
+++ b/lib/mrsk/cli/healthcheck.rb
@@ -1,8 +1,4 @@
 class Mrsk::Cli::Healthcheck < Mrsk::Cli::Base
-  MAX_ATTEMPTS = 7
-
-  class HealthcheckError < StandardError; end
-
  default_command :perform

  desc "perform", "Health check current app version"
@@ -10,37 +6,11 @@ class Mrsk::Cli::Healthcheck < Mrsk::Cli::Base
    on(MRSK.primary_host) do
      begin
        execute *MRSK.healthcheck.run
-
-        target = "Health check against #{MRSK.config.healthcheck["path"]}"
-        attempt = 1
-
-        begin
-          status = capture_with_info(*MRSK.healthcheck.curl)
-
-          if status == "200"
-            info "#{target} succeeded with 200 OK!"
-          else
-            raise HealthcheckError, "#{target} failed with status #{status}"
-          end
-        rescue SSHKit::Command::Failed
-          if attempt <= MAX_ATTEMPTS
-            info "#{target} failed to respond, retrying in #{attempt}s..."
-            sleep attempt
-            attempt += 1
-
-            retry
-          else
-            raise
-          end
-        end
-      rescue SSHKit::Command::Failed, HealthcheckError => e
+        Mrsk::Utils::HealthcheckPoller.wait_for_healthy { capture_with_info(*MRSK.healthcheck.status) }
+      rescue Mrsk::Utils::HealthcheckPoller::HealthcheckError => e
        error capture_with_info(*MRSK.healthcheck.logs)
-
-        if e.message =~ /curl/
-          raise SSHKit::Command::Failed, "#{target} failed to return 200 OK!"
-        else
+        error capture_with_pretty_json(*MRSK.healthcheck.container_health_log)
        raise
-        end
      ensure
        execute *MRSK.healthcheck.stop, raise_on_non_zero_exit: false
        execute *MRSK.healthcheck.remove, raise_on_non_zero_exit: false
--- a/lib/mrsk/cli/lock.rb
+++ b/lib/mrsk/cli/lock.rb
@@ -12,7 +12,7 @@ class Mrsk::Cli::Lock < Mrsk::Cli::Base
    message = options[:message]
    handle_missing_lock do
      on(MRSK.primary_host) { execute *MRSK.lock.acquire(message, MRSK.config.version) }
-      say "Set the deploy lock"
+      say "Acquired the deploy lock"
    end
  end

@@ -20,7 +20,7 @@ class Mrsk::Cli::Lock < Mrsk::Cli::Base
  def release
    handle_missing_lock do
      on(MRSK.primary_host) { execute *MRSK.lock.release }
-      say "Removed the deploy lock"
+      say "Released the deploy lock"
    end
  end

--- a/lib/mrsk/cli/main.rb
+++ b/lib/mrsk/cli/main.rb
@@ -17,9 +17,6 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
      invoke_options = deploy_options

      runtime = print_runtime do
-        say "Ensure curl and Docker are installed...", :magenta
-        invoke "mrsk:cli:server:bootstrap", [], invoke_options
-
        say "Log into image registry...", :magenta
        invoke "mrsk:cli:registry:login", [], invoke_options

@@ -37,7 +34,12 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
        say "Ensure app can pass healthcheck...", :magenta
        invoke "mrsk:cli:healthcheck:perform", [], invoke_options

+        say "Detect stale containers...", :magenta
+        invoke "mrsk:cli:app:stale_containers", [], invoke_options
+
+        hold_lock_on_error do
          invoke "mrsk:cli:app:boot", [], invoke_options
+        end

        say "Prune old containers and images...", :magenta
        invoke "mrsk:cli:prune:all", [], invoke_options
@@ -65,8 +67,13 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
        say "Ensure app can pass healthcheck...", :magenta
        invoke "mrsk:cli:healthcheck:perform", [], invoke_options

+        say "Detect stale containers...", :magenta
+        invoke "mrsk:cli:app:stale_containers", [], invoke_options
+
+        hold_lock_on_error do
          invoke "mrsk:cli:app:boot", [], invoke_options
        end
+      end

      audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
    end
@@ -75,23 +82,34 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
  desc "rollback [VERSION]", "Rollback app to VERSION"
  def rollback(version)
    with_lock do
+      invoke_options = deploy_options
+
+      hold_lock_on_error do
        MRSK.config.version = version
-
-      if container_name_available?(MRSK.config.service_with_version)
-        say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta
-
-        cli = self
        old_version = nil

-        on(MRSK.hosts) do |host|
-          old_version = capture_with_info(*MRSK.app.current_running_version).strip.presence
+        if container_available?(version)
+          say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta

-          execute *MRSK.app.start
+          on(MRSK.hosts) do
+            execute *MRSK.auditor.record("Tagging #{MRSK.config.absolute_image} as the latest image"), verbosity: :debug
+            execute *MRSK.app.tag_current_as_latest
+          end
+
+          on(MRSK.hosts) do |host|
+            roles = MRSK.roles_on(host)
+
+            roles.each do |role|
+              app = MRSK.app(role: role)
+              old_version = capture_with_info(*app.current_running_version).strip.presence
+
+              execute *app.start

              if old_version
                sleep MRSK.config.readiness_delay

-            execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false
+                execute *app.stop(version: old_version), raise_on_non_zero_exit: false
+              end
            end
          end

@@ -101,6 +119,7 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
        end
      end
    end
+  end

  desc "details", "Show details about all containers"
  def details
@@ -119,7 +138,7 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
  desc "config", "Show combined config (including secrets!)"
  def config
    run_locally do
-      puts MRSK.config.to_h.to_yaml
+      puts Mrsk::Utils.redacted(MRSK.config.to_h).to_yaml
    end
  end

@@ -181,6 +200,13 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
    end
  end

+  desc "broadcast", "Broadcast an audit message"
+  option :message, aliases: "-m", type: :string, desc: "Audit mesasge", required: true
+  def broadcast
+    say "Broadcast: #{options[:message]}", :magenta
+    audit_broadcast options[:message]
+  end
+
  desc "version", "Show MRSK version"
  def version
    puts Mrsk::VERSION
@@ -214,10 +240,24 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
  subcommand "lock", Mrsk::Cli::Lock

  private
-    def container_name_available?(container_name, host: MRSK.primary_host)
-      container_names = nil
-      on(host) { container_names = capture_with_info(*MRSK.app.list_container_names).split("\n") }
-      Array(container_names).include?(container_name)
+    def container_available?(version)
+      begin
+        on(MRSK.hosts) do
+          MRSK.roles_on(host).each do |role|
+            container_id = capture_with_info(*MRSK.app(role: role).container_id_for_version(version))
+            raise "Container not found" unless container_id.present?
+          end
+        end
+      rescue SSHKit::Runner::ExecuteError => e
+        if e.message =~ /Container not found/
+          say "Error looking for container version #{version}: #{e.message}"
+          return false
+        else
+          raise
+        end
+      end
+
+      true
    end

    def deploy_options
--- a/lib/mrsk/cli/prune.rb
+++ b/lib/mrsk/cli/prune.rb
@@ -7,7 +7,7 @@ class Mrsk::Cli::Prune < Mrsk::Cli::Base
    end
  end

-  desc "images", "Prune unused images older than 7 days"
+  desc "images", "Prune dangling images"
  def images
    with_lock do
      on(MRSK.hosts) do
@@ -17,7 +17,7 @@ class Mrsk::Cli::Prune < Mrsk::Cli::Base
    end
  end

-  desc "containers", "Prune stopped containers older than 3 days"
+  desc "containers", "Prune all stopped containers, except the last 5"
  def containers
    with_lock do
      on(MRSK.hosts) do
--- a/lib/mrsk/cli/server.rb
+++ b/lib/mrsk/cli/server.rb
@@ -1,17 +1,21 @@
 class Mrsk::Cli::Server < Mrsk::Cli::Base
-  desc "bootstrap", "Ensure curl and Docker are installed on servers"
+  desc "bootstrap", "Set up Docker to run MRSK apps"
  def bootstrap
-    with_lock do
-      on(MRSK.hosts + MRSK.accessory_hosts) do
-        dependencies_to_install = Array.new.tap do |dependencies|
-          dependencies << "curl" unless execute "which curl", raise_on_non_zero_exit: false
-          dependencies << "docker.io" unless execute "which docker", raise_on_non_zero_exit: false
+    missing = []
+
+    on(MRSK.hosts | MRSK.accessory_hosts) do |host|
+      unless execute(*MRSK.docker.installed?, raise_on_non_zero_exit: false)
+        if execute(*MRSK.docker.superuser?, raise_on_non_zero_exit: false)
+          info "Missing Docker on #{host}. Installing…"
+          execute *MRSK.docker.install
+        else
+          missing << host
+        end
+      end
    end

-        if dependencies_to_install.any?
-          execute "apt-get update -y && apt-get install #{dependencies_to_install.join(" ")} -y"
-        end
-      end
+    if missing.any?
+      raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/"
    end
  end
 end
--- a/lib/mrsk/cli/traefik.rb
+++ b/lib/mrsk/cli/traefik.rb
@@ -2,7 +2,10 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
  desc "boot", "Boot Traefik on servers"
  def boot
    with_lock do
-      on(MRSK.traefik_hosts) { execute *MRSK.traefik.run, raise_on_non_zero_exit: false }
+      on(MRSK.traefik_hosts) do
+        execute *MRSK.registry.login
+        execute *MRSK.traefik.run, raise_on_non_zero_exit: false
+      end
    end
  end

@@ -91,7 +94,7 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
    end
  end

-  desc "remove_container", "Remove Traefik image from servers", hide: true
+  desc "remove_image", "Remove Traefik image from servers", hide: true
  def remove_image
    with_lock do
      on(MRSK.traefik_hosts) do
--- a/lib/mrsk/commander.rb
+++ b/lib/mrsk/commander.rb
@@ -2,11 +2,12 @@ require "active_support/core_ext/enumerable"
 require "active_support/core_ext/module/delegation"

 class Mrsk::Commander
-  attr_accessor :verbosity, :lock_count
+  attr_accessor :verbosity, :holding_lock, :hold_lock_on_error

  def initialize
    self.verbosity = :info
-    self.lock_count = 0
+    self.holding_lock = false
+    self.hold_lock_on_error = false
  end

  def config
@@ -35,7 +36,7 @@ class Mrsk::Commander
  end

  def primary_host
-    specific_hosts&.first || config.primary_web_host
+    specific_hosts&.first || specific_roles&.first&.primary_host || config.primary_web_host
  end

  def roles
@@ -50,6 +51,14 @@ class Mrsk::Commander
    end
  end

+  def boot_strategy
+    if config.boot.limit.present?
+      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
+    else
+      {}
+    end
+  end
+
  def roles_on(host)
    roles.select { |role| role.hosts.include?(host.to_s) }.map(&:name)
  end
@@ -75,14 +84,18 @@ class Mrsk::Commander
    Mrsk::Commands::Accessory.new(config, name: name)
  end

-  def auditor(role: nil)
-    Mrsk::Commands::Auditor.new(config, role: role)
+  def auditor(**details)
+    Mrsk::Commands::Auditor.new(config, **details)
  end

  def builder
    @builder ||= Mrsk::Commands::Builder.new(config)
  end

+  def docker
+    @docker ||= Mrsk::Commands::Docker.new(config)
+  end
+
  def healthcheck
    @healthcheck ||= Mrsk::Commands::Healthcheck.new(config)
  end
@@ -115,6 +128,14 @@ class Mrsk::Commander
    SSHKit.config.output_verbosity = old_level
  end

+  def holding_lock?
+    self.holding_lock
+  end
+
+  def hold_lock_on_error?
+    self.hold_lock_on_error
+  end
+
  private
    # Lazy setup of SSHKit
    def configure_sshkit_with(config)
--- a/lib/mrsk/commands/app.rb
+++ b/lib/mrsk/commands/app.rb
@@ -15,6 +15,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
      "--name", container_name,
      "-e", "MRSK_CONTAINER_NAME=\"#{container_name}\"",
      *role.env_args,
+      *role.health_check_args,
      *config.logging_args,
      *config.volume_args,
      *role.label_args,
@@ -27,9 +28,13 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
    docker :start, container_name
  end

+  def status(version:)
+    pipe container_id_for_version(version), xargs(docker(:inspect, "--format", DOCKER_HEALTH_STATUS_FORMAT))
+  end
+
  def stop(version: nil)
    pipe \
-      version ? container_id_for_version(version) : current_container_id,
+      version ? container_id_for_version(version) : current_running_container_id,
      xargs(config.stop_wait_time ? docker(:stop, "-t", config.stop_wait_time) : docker(:stop))
  end

@@ -40,7 +45,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base

  def logs(since: nil, lines: nil, grep: nil)
    pipe \
-      current_container_id,
+      current_running_container_id,
      "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
      ("grep '#{grep}'" if grep)
  end
@@ -48,7 +53,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
  def follow_logs(host:, grep: nil)
    run_over_ssh \
      pipe(
-        current_container_id,
+        current_running_container_id,
        "xargs docker logs --timestamps --tail 10 --follow 2>&1",
        (%(grep "#{grep}") if grep)
      ),
@@ -82,8 +87,8 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
  end


-  def current_container_id
-    docker :ps, "--quiet", *filter_args
+  def current_running_container_id
+    docker :ps, "--quiet", *filter_args(status: :running), "--latest"
  end

  def container_id_for_version(version)
@@ -91,11 +96,14 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
  end

  def current_running_version
-    # FIXME: Find more graceful way to extract the version from "app-version" than using sed and tail!
+    list_versions("--latest", status: :running)
+  end
+
+  def list_versions(*docker_args, status: nil)
    pipe \
-      docker(:ps, *filter_args, "--format", '"{{.Names}}"'),
-      %(sed 's/-/\\n/g'),
-      "tail -n 1"
+      docker(:ps, *filter_args(status: status), *docker_args, "--format", '"{{.Names}}"'),
+      %(grep -oE "\\-[^-]+$"), # Extract SHA from "service-role-dest-SHA"
+      %(cut -c 2-)
  end

  def list_containers
@@ -128,20 +136,25 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
    docker :image, :prune, "--all", "--force", *filter_args
  end

+  def tag_current_as_latest
+    docker :tag, config.absolute_image, config.latest_image
+  end
+

  private
    def container_name(version = nil)
      [ config.service, role, config.destination, version || config.version ].compact.join("-")
    end

-    def filter_args
-      argumentize "--filter", filters
+    def filter_args(status: nil)
+      argumentize "--filter", filters(status: status)
    end

-    def filters
+    def filters(status: nil)
      [ "label=service=#{config.service}" ].tap do |filters|
        filters << "label=destination=#{config.destination}" if config.destination
        filters << "label=role=#{role}" if role
+        filters << "status=#{status}" if status
      end
    end
 end
--- a/lib/mrsk/commands/auditor.rb
+++ b/lib/mrsk/commands/auditor.rb
@@ -1,24 +1,24 @@
-require "active_support/core_ext/time/conversions"
+require "time"

 class Mrsk::Commands::Auditor < Mrsk::Commands::Base
-  attr_reader :role
+  attr_reader :details

-  def initialize(config, role: nil)
+  def initialize(config, **details)
    super(config)
-    @role = role
+    @details = default_details.merge(details)
  end

  # Runs remotely
-  def record(line)
+  def record(line, **details)
    append \
-      [ :echo, tagged_record_line(line) ],
+      [ :echo, *audit_tags(**details), line ],
      audit_log_file
  end

  # Runs locally
-  def broadcast(line)
+  def broadcast(line, **details)
    if broadcast_cmd = config.audit_broadcast_cmd
-      [ broadcast_cmd, tagged_broadcast_line(line) ]
+      [ broadcast_cmd, *broadcast_args(line, **details), env: env_for(event: line, **details) ]
    end
  end

@@ -31,27 +31,29 @@ class Mrsk::Commands::Auditor < Mrsk::Commands::Base
      [ "mrsk", config.service, config.destination, "audit.log" ].compact.join("-")
    end

-    def tagged_record_line(line)
-      tagged_line recorded_at_tag, performer_tag, role_tag, line
+    def default_details
+      { recorded_at: Time.now.utc.iso8601,
+        performer: `whoami`.chomp,
+        destination: config.destination }
    end

-    def tagged_broadcast_line(line)
-      tagged_line performer_tag, role_tag, line
+    def audit_tags(**details)
+      tags_for **self.details.merge(details)
    end

-    def tagged_line(*tags_and_line)
-      "'#{tags_and_line.compact.join(" ")}'"
+    def broadcast_args(line, **details)
+      "'#{broadcast_tags(**details).join(" ")} #{line}'"
    end

-    def recorded_at_tag
-      "[#{Time.now.to_fs(:db)}]"
+    def broadcast_tags(**details)
+      tags_for **self.details.merge(details).except(:recorded_at)
    end

-    def performer_tag
-      "[#{`whoami`.strip}]"
+    def tags_for(**details)
+      details.compact.values.map { |value| "[#{value}]" }
    end

-    def role_tag
-      "[#{role}]" if role
+    def env_for(**details)
+      self.details.merge(details).compact.transform_keys { |detail| "MRSK_#{detail.upcase}" }
    end
 end
--- a/lib/mrsk/commands/base.rb
+++ b/lib/mrsk/commands/base.rb
@@ -1,6 +1,9 @@
 module Mrsk::Commands
  class Base
-    delegate :redact, :argumentize, to: Mrsk::Utils
+    delegate :sensitive, :argumentize, to: Mrsk::Utils
+
+    DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'"
+    DOCKER_HEALTH_LOG_FORMAT    = "'{{json .State.Health}}'"

    attr_accessor :config

--- a/lib/mrsk/commands/builder.rb
+++ b/lib/mrsk/commands/builder.rb
@@ -2,7 +2,7 @@ class Mrsk::Commands::Builder < Mrsk::Commands::Base
  delegate :create, :remove, :push, :clean, :pull, :info, to: :target

  def name
-    target.class.to_s.remove("Mrsk::Commands::Builder::").underscore
+    target.class.to_s.remove("Mrsk::Commands::Builder::").underscore.inquiry
  end

  def target
@@ -33,4 +33,24 @@ class Mrsk::Commands::Builder < Mrsk::Commands::Base
  def multiarch_remote
    @multiarch_remote ||= Mrsk::Commands::Builder::Multiarch::Remote.new(config)
  end
+
+
+  def ensure_local_dependencies_installed
+    if name.native?
+      ensure_local_docker_installed
+    else
+      combine \
+        ensure_local_docker_installed,
+        ensure_local_buildx_installed
+    end
+  end
+
+  private
+    def ensure_local_docker_installed
+      docker "--version"
+    end
+
+    def ensure_local_buildx_installed
+      docker :buildx, "version"
+    end
 end
--- a/lib/mrsk/commands/builder/base.rb
+++ b/lib/mrsk/commands/builder/base.rb
@@ -1,4 +1,7 @@
+
 class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
+  class BuilderError < StandardError; end
+
  delegate :argumentize, to: Mrsk::Utils

  def clean
@@ -7,7 +10,6 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base

  def pull
    docker :pull, config.absolute_image
-    docker :pull, config.latest_image
  end

  def build_options
@@ -18,6 +20,7 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
    context
  end

+
  private
    def build_tags
      [ "-t", config.absolute_image, "-t", config.latest_image ]
@@ -28,7 +31,7 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
    end

    def build_args
-      argumentize "--build-arg", args, redacted: true
+      argumentize "--build-arg", args, sensitive: true
    end

    def build_secrets
@@ -36,7 +39,11 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
    end

    def build_dockerfile
+      if Pathname.new(File.expand_path(dockerfile)).exist?
        argumentize "--file", dockerfile
+      else
+        raise BuilderError, "Missing #{dockerfile}"
+      end
    end

    def args
--- a/lib/mrsk/commands/docker.rb
+++ b/lib/mrsk/commands/docker.rb
@@ -0,0 +1,21 @@
+class Mrsk::Commands::Docker < Mrsk::Commands::Base
+  # Install Docker using the https://github.com/docker/docker-install convenience script.
+  def install
+    pipe [ :curl, "-fsSL", "https://get.docker.com" ], :sh
+  end
+
+  # Checks the Docker client version. Fails if Docker is not installed.
+  def installed?
+    docker "-v"
+  end
+
+  # Checks the Docker server version. Fails if Docker is not running.
+  def running?
+    docker :version
+  end
+
+  # Do we have superuser access to install Docker and start system services?
+  def superuser?
+    [ '[ "${EUID:-$(id -u)}" -eq 0 ]' ]
+  end
+end
--- a/lib/mrsk/commands/healthcheck.rb
+++ b/lib/mrsk/commands/healthcheck.rb
@@ -11,14 +11,19 @@ class Mrsk::Commands::Healthcheck < Mrsk::Commands::Base
      "--label", "service=#{container_name}",
      "-e", "MRSK_CONTAINER_NAME=\"#{container_name}\"",
      *web.env_args,
+      *web.health_check_args,
      *config.volume_args,
      *web.option_args,
      config.absolute_image,
      web.cmd
  end

-  def curl
-    [ :curl, "--silent", "--output", "/dev/null", "--write-out", "'%{http_code}'", "--max-time", "2", health_url ]
+  def status
+    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_STATUS_FORMAT))
+  end
+
+  def container_health_log
+    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT))
  end

  def logs
--- a/lib/mrsk/commands/lock.rb
+++ b/lib/mrsk/commands/lock.rb
@@ -1,5 +1,5 @@
 require "active_support/duration"
-require "active_support/core_ext/numeric/time"
+require "time"

 class Mrsk::Commands::Lock < Mrsk::Commands::Base
  def acquire(message, version)
@@ -49,7 +49,7 @@ class Mrsk::Commands::Lock < Mrsk::Commands::Base

    def lock_details(message, version)
      <<~DETAILS.strip
-        Locked by: #{locked_by} at #{Time.now.gmtime}
+        Locked by: #{locked_by} at #{Time.now.utc.iso8601}
        Version: #{version}
        Message: #{message}
      DETAILS
--- a/lib/mrsk/commands/prune.rb
+++ b/lib/mrsk/commands/prune.rb
@@ -2,11 +2,19 @@ require "active_support/duration"
 require "active_support/core_ext/numeric/time"

 class Mrsk::Commands::Prune < Mrsk::Commands::Base
-  def images(until_hours: 7.days.in_hours.to_i)
-    docker :image, :prune, "--all", "--force", "--filter", "label=service=#{config.service}", "--filter", "until=#{until_hours}h"
+  def images
+    docker :image, :prune, "--force", "--filter", "label=service=#{config.service}", "--filter", "dangling=true"
  end

-  def containers(until_hours: 3.days.in_hours.to_i)
-    docker :container, :prune, "--force", "--filter", "label=service=#{config.service}", "--filter", "until=#{until_hours}h"
+  def containers(keep_last: 5)
+    pipe \
+      docker(:ps, "-q", "-a", "--filter", "label=service=#{config.service}", *stopped_containers_filters),
+      "tail -n +#{keep_last + 1}",
+      "while read container_id; do docker rm $container_id; done"
+  end
+
+  private
+    def stopped_containers_filters
+      [ "created", "exited", "dead" ].flat_map { |status| ["--filter", "status=#{status}"] }
    end
 end
--- a/lib/mrsk/commands/registry.rb
+++ b/lib/mrsk/commands/registry.rb
@@ -2,7 +2,7 @@ class Mrsk::Commands::Registry < Mrsk::Commands::Base
  delegate :registry, to: :config

  def login
-    docker :login, registry["server"], "-u", redact(lookup("username")), "-p", redact(lookup("password"))
+    docker :login, registry["server"], "-u", sensitive(lookup("username")), "-p", sensitive(lookup("password"))
  end

  def logout
--- a/lib/mrsk/commands/traefik.rb
+++ b/lib/mrsk/commands/traefik.rb
@@ -1,7 +1,7 @@
 class Mrsk::Commands::Traefik < Mrsk::Commands::Base
-  delegate :optionize, to: Mrsk::Utils
+  delegate :argumentize, :optionize, to: Mrsk::Utils

-  IMAGE = "traefik:v2.9.9"
+  DEFAULT_IMAGE = "traefik:v2.9"
  CONTAINER_PORT = 80

  def run
@@ -11,8 +11,9 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
      "--publish", port,
      "--volume", "/var/run/docker.sock:/var/run/docker.sock",
      *config.logging_args,
+      *label_args,
      *docker_options_args,
-      IMAGE,
+      image,
      "--providers.docker",
      "--log.level=DEBUG",
      *cmd_option_args
@@ -56,6 +57,18 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
  end

  private
+    def label_args
+      argumentize "--label", labels
+    end
+
+    def labels
+      config.traefik["labels"] || []
+    end
+
+    def image
+      config.traefik.fetch("image") { DEFAULT_IMAGE }
+    end
+
    def docker_options_args
      optionize(config.traefik["options"] || {})
    end
--- a/lib/mrsk/configuration.rb
+++ b/lib/mrsk/configuration.rb
@@ -87,6 +87,10 @@ class Mrsk::Configuration
    roles.select(&:running_traefik?).flat_map(&:hosts).uniq
  end

+  def boot
+    Mrsk::Configuration::Boot.new(config: self)
+  end
+

  def repository
    [ raw_config.registry["server"], image ].compact.join("/")
@@ -143,6 +147,8 @@ class Mrsk::Configuration
    if raw_config.ssh.present? && raw_config.ssh["proxy"]
      Net::SSH::Proxy::Jump.new \
        raw_config.ssh["proxy"].include?("@") ? raw_config.ssh["proxy"] : "root@#{raw_config.ssh["proxy"]}"
+    elsif raw_config.ssh.present? && raw_config.ssh["proxy_command"]
+      Net::SSH::Proxy::Command.new(raw_config.ssh["proxy_command"])
    end
  end

@@ -156,7 +162,7 @@ class Mrsk::Configuration
  end

  def healthcheck
-    { "path" => "/up", "port" => 3000 }.merge(raw_config.healthcheck || {})
+    { "path" => "/up", "port" => 3000, "max_attempts" => 7 }.merge(raw_config.healthcheck || {})
  end

  def readiness_delay
--- a/lib/mrsk/configuration/boot.rb
+++ b/lib/mrsk/configuration/boot.rb
@@ -0,0 +1,20 @@
+class Mrsk::Configuration::Boot
+  def initialize(config:)
+    @options = config.raw_config.boot || {}
+    @host_count = config.all_hosts.count
+  end
+
+  def limit
+    limit = @options["limit"]
+
+    if limit.to_s.end_with?("%")
+      @host_count * limit.to_i / 100
+    else
+      limit
+    end
+  end
+
+  def wait
+    @options["wait"]
+  end
+end
--- a/lib/mrsk/configuration/role.rb
+++ b/lib/mrsk/configuration/role.rb
@@ -35,6 +35,21 @@ class Mrsk::Configuration::Role
    argumentize_env_with_secrets env
  end

+  def health_check_args
+    if health_check_cmd.present?
+      optionize({ "health-cmd" => health_check_cmd, "health-interval" => "1s" })
+    else
+      []
+    end
+  end
+
+  def health_check_cmd
+    options = specializations["healthcheck"] || {}
+    options = config.healthcheck.merge(options) if running_traefik?
+
+    options["cmd"] || http_health_check(port: options["port"], path: options["path"])
+  end
+
  def cmd
    specializations["cmd"]
  end
@@ -74,18 +89,23 @@ class Mrsk::Configuration::Role
    def traefik_labels
      if running_traefik?
        {
-          "traefik.http.routers.#{config.service}.rule" => "PathPrefix(`/`)",
-          "traefik.http.services.#{config.service}.loadbalancer.healthcheck.path" => config.healthcheck["path"],
-          "traefik.http.services.#{config.service}.loadbalancer.healthcheck.interval" => "1s",
-          "traefik.http.middlewares.#{config.service}-retry.retry.attempts" => "5",
-          "traefik.http.middlewares.#{config.service}-retry.retry.initialinterval" => "500ms",
-          "traefik.http.routers.#{config.service}.middlewares" => "#{config.service}-retry@docker"
+          # Setting a service property ensures that the generated service name will be consistent between versions
+          "traefik.http.services.#{traefik_service}.loadbalancer.server.scheme" => "http",
+
+          "traefik.http.routers.#{traefik_service}.rule" => "PathPrefix(`/`)",
+          "traefik.http.middlewares.#{traefik_service}-retry.retry.attempts" => "5",
+          "traefik.http.middlewares.#{traefik_service}-retry.retry.initialinterval" => "500ms",
+          "traefik.http.routers.#{traefik_service}.middlewares" => "#{traefik_service}-retry@docker"
        }
      else
        {}
      end
    end

+    def traefik_service
+      [ config.service, name, config.destination ].compact.join("-")
+    end
+
    def custom_labels
      Hash.new.tap do |labels|
        labels.merge!(config.labels) if config.labels.present?
@@ -121,4 +141,8 @@ class Mrsk::Configuration::Role
        new_env["clear"] = (clear_app_env + clear_role_env).uniq
      end
    end
+
+    def http_health_check(port:, path:)
+      "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present?
+    end
 end
--- a/lib/mrsk/sshkit_with_ext.rb
+++ b/lib/mrsk/sshkit_with_ext.rb
@@ -1,12 +1,52 @@
 require "sshkit"
 require "sshkit/dsl"
+require "active_support/core_ext/hash/deep_merge"
+require "json"

 class SSHKit::Backend::Abstract
-  def capture_with_info(*args)
-    capture(*args, verbosity: Logger::INFO)
+  def capture_with_info(*args, **kwargs)
+    capture(*args, **kwargs, verbosity: Logger::INFO)
+  end
+
+  def capture_with_pretty_json(*args, **kwargs)
+    JSON.pretty_generate(JSON.parse(capture(*args, **kwargs)))
  end

  def puts_by_host(host, output, type: "App")
    puts "#{type} Host: #{host}\n#{output}\n\n"
  end
+
+  # Our execution pattern is for the CLI execute args lists returned
+  # from commands, but this doesn't support returning execution options
+  # from the command.
+  #
+  # Support this by using kwargs for CLI options and merging with the
+  # args-extracted options.
+  module CommandEnvMerge
+    private
+
+    # Override to merge options returned by commands in the args list with
+    # options passed by the CLI and pass them along as kwargs.
+    def command(args, options)
+      more_options, args = args.partition { |a| a.is_a? Hash }
+      more_options << options
+
+      build_command(args, **more_options.reduce(:deep_merge))
+    end
+
+    # Destructure options to pluck out env for merge
+    def build_command(args, env: nil, **options)
+      # Rely on native Ruby kwargs precedence rather than explicit Hash merges
+      SSHKit::Command.new(*args, **default_command_options, **options, env: env_for(env))
+    end
+
+    def default_command_options
+      { in: pwd_path, host: @host, user: @user, group: @group }
+    end
+
+    def env_for(env)
+      @env.to_h.merge(env.to_h)
+    end
+  end
+  prepend CommandEnvMerge
 end
--- a/lib/mrsk/utils.rb
+++ b/lib/mrsk/utils.rb
@@ -1,12 +1,15 @@
 module Mrsk::Utils
  extend self

+  DOLLAR_SIGN_WITHOUT_SHELL_EXPANSION_REGEX = /\$(?!{[^\}]*\})/
+
  # Return a list of escaped shell arguments using the same named argument against the passed attributes (hash or array).
-  def argumentize(argument, attributes, redacted: false)
+  def argumentize(argument, attributes, sensitive: false)
    Array(attributes).flat_map do |key, value|
      if value.present?
-        escaped_pair = [ key, escape_shell_value(value) ].join("=")
-        [ argument, redacted ? redact(escaped_pair) : escaped_pair ]
+        attr = "#{key}=#{escape_shell_value(value)}"
+        attr = self.sensitive(attr, redaction: "#{key}=[REDACTED]") if sensitive
+        [ argument, attr]
      else
        [ argument, key ]
      end
@@ -17,7 +20,7 @@ module Mrsk::Utils
  # but redacts and expands secrets.
  def argumentize_env_with_secrets(env)
    if (secrets = env["secret"]).present?
-      argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, redacted: true) + argumentize("-e", env["clear"])
+      argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, sensitive: true) + argumentize("-e", env["clear"])
    else
      argumentize "-e", env.fetch("clear", env)
    end
@@ -39,14 +42,44 @@ module Mrsk::Utils
    args.flat_map { |key, value| value.try(:map) { |entry| [key, entry] } || [ [ key, value ] ] }
  end

-  # Copied from SSHKit::Backend::Abstract#redact to be available inside Commands classes
-  def redact(arg) # Used in execute_command to hide redact() args a user passes in
-    arg.to_s.extend(SSHKit::Redaction) # to_s due to our inability to extend Integer, etc
+  # Marks sensitive values for redaction in logs and human-visible output.
+  # Pass `redaction:` to change the default `"[REDACTED]"` redaction, e.g.
+  # `sensitive "#{arg}=#{secret}", redaction: "#{arg}=xxxx"
+  def sensitive(...)
+    Mrsk::Utils::Sensitive.new(...)
+  end
+
+  def redacted(value)
+    case
+    when value.respond_to?(:redaction)
+      value.redaction
+    when value.respond_to?(:transform_values)
+      value.transform_values { |value| redacted value }
+    when value.respond_to?(:map)
+      value.map { |element| redacted element }
+    else
+      value
+    end
+  end
+
+  def unredacted(value)
+    case
+    when value.respond_to?(:unredacted)
+      value.unredacted
+    when value.respond_to?(:transform_values)
+      value.transform_values { |value| unredacted value }
+    when value.respond_to?(:map)
+      value.map { |element| unredacted element }
+    else
+      value
+    end
  end

  # Escape a value to make it safe for shell use.
  def escape_shell_value(value)
-    value.to_s.dump.gsub(/`/, '\\\\`')
+    value.to_s.dump
+      .gsub(/`/, '\\\\`')
+      .gsub(DOLLAR_SIGN_WITHOUT_SHELL_EXPANSION_REGEX, '\$')
  end

  # Abbreviate a git revhash for concise display
--- a/lib/mrsk/utils/healthcheck_poller.rb
+++ b/lib/mrsk/utils/healthcheck_poller.rb
@@ -0,0 +1,39 @@
+class Mrsk::Utils::HealthcheckPoller
+  TRAEFIK_HEALTHY_DELAY = 2
+
+  class HealthcheckError < StandardError; end
+
+  class << self
+    def wait_for_healthy(pause_after_ready: false, &block)
+      attempt = 1
+      max_attempts = MRSK.config.healthcheck["max_attempts"]
+
+      begin
+        case status = block.call
+        when "healthy"
+          sleep TRAEFIK_HEALTHY_DELAY if pause_after_ready
+        when "running" # No health check configured
+          sleep MRSK.config.readiness_delay if pause_after_ready
+        else
+          raise HealthcheckError, "container not ready (#{status})"
+        end
+      rescue HealthcheckError => e
+        if attempt <= max_attempts
+          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+          sleep attempt
+          attempt += 1
+          retry
+        else
+          raise
+        end
+      end
+
+      info "Container is healthy!"
+    end
+
+    private
+      def info(message)
+        SSHKit.config.output.info(message)
+      end
+  end
+end
--- a/lib/mrsk/utils/sensitive.rb
+++ b/lib/mrsk/utils/sensitive.rb
@@ -0,0 +1,19 @@
+require "active_support/core_ext/module/delegation"
+
+class Mrsk::Utils::Sensitive
+  # So SSHKit knows to redact these values.
+  include SSHKit::Redaction
+
+  attr_reader :unredacted, :redaction
+  delegate :to_s, to: :unredacted
+  delegate :inspect, to: :redaction
+
+  def initialize(value, redaction: "[REDACTED]")
+    @unredacted, @redaction = value, redaction
+  end
+
+  # Sensitive values won't leak into YAML output.
+  def encode_with(coder)
+    coder.represent_scalar nil, redaction
+  end
+end
--- a/lib/mrsk/version.rb
+++ b/lib/mrsk/version.rb
@@ -1,3 +1,3 @@
 module Mrsk
-  VERSION = "0.10.1"
+  VERSION = "0.12.1"
 end
--- a/mrsk.gemspec
+++ b/mrsk.gemspec
@@ -14,6 +14,7 @@ Gem::Specification.new do |spec|

  spec.add_dependency "activesupport", ">= 7.0"
  spec.add_dependency "sshkit", "~> 1.21"
+  spec.add_dependency "net-ssh", "~> 7.0"
  spec.add_dependency "thor", "~> 1.2"
  spec.add_dependency "dotenv", "~> 2.8"
  spec.add_dependency "zeitwerk", "~> 2.5"
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -2,10 +2,14 @@ require_relative "cli_test_case"

 class CliAppTest < CliTestCase
  test "boot" do
-    # Stub current version fetch
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture).returns("123") # old version
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+    .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+    .returns("running") # health check

    run_command("boot").tap do |output|
+      assert_match "docker tag dhh/app:latest dhh/app:latest", output
      assert_match "docker run --detach --restart unless-stopped", output
      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
    end
@@ -15,11 +19,15 @@ class CliAppTest < CliTestCase
    run_command("details") # Preheat MRSK const

    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet")
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
      .returns("12345678") # running version

    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1")
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
      .returns("123") # old version

    run_command("boot").tap do |output|
@@ -32,6 +40,16 @@ class CliAppTest < CliTestCase
    Thread.report_on_exception = true
  end

+  test "boot uses group strategy when specified" do
+    Mrsk::Cli::App.any_instance.stubs(:on).with("1.1.1.1").twice # acquire & release lock
+    Mrsk::Cli::App.any_instance.stubs(:on).with([ "1.1.1.1" ]) # tag container
+
+    # Strategy is used when booting the containers
+    Mrsk::Cli::App.any_instance.expects(:on).with([ "1.1.1.1" ], in: :groups, limit: 3, wait: 2).with_block_given
+
+    run_command("boot", config: :with_boot_strategy)
+  end
+
  test "start" do
    run_command("start").tap do |output|
      assert_match "docker start app-web-999", output
@@ -40,7 +58,28 @@ class CliAppTest < CliTestCase

  test "stop" do
    run_command("stop").tap do |output|
-      assert_match "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop", output
+      assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop", output
+    end
+  end
+
+  test "stale_containers" do
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
+      .returns("12345678\n87654321")
+
+    run_command("stale_containers").tap do |output|
+      assert_match /Detected stale container for role web with version 87654321/, output
+    end
+  end
+
+  test "stop stale_containers" do
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
+      .returns("12345678\n87654321")
+
+    run_command("stale_containers", "--stop").tap do |output|
+      assert_match /Stopping stale container for role web with version 87654321/, output
+      assert_match /#{Regexp.escape("docker container ls --all --filter name=^app-web-87654321$ --quiet | xargs docker stop")}/, output
    end
  end

@@ -52,7 +91,7 @@ class CliAppTest < CliTestCase

  test "remove" do
    run_command("remove").tap do |output|
-      assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop")}/, output
+      assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop")}/, output
      assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output
      assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output
    end
@@ -84,7 +123,7 @@ class CliAppTest < CliTestCase

  test "exec with reuse" do
    run_command("exec", "--reuse", "ruby -v").tap do |output|
-      assert_match "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", output # Get current version
+      assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output # Get current version
      assert_match "docker exec app-web-999 ruby -v", output
    end
  end
@@ -103,33 +142,33 @@ class CliAppTest < CliTestCase

  test "logs" do
    SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 2>&1'")
+      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest| xargs docker logs --timestamps --tail 10 2>&1'")

-    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --tail 100 2>&1", run_command("logs")
+    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --tail 100 2>&1", run_command("logs")
  end

  test "logs with follow" do
    SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
+      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1'")

-    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
+    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
  end

  test "version" do
    run_command("version").tap do |output|
-      assert_match "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", output
+      assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output
    end
  end


  test "version through main" do
    stdouted { Mrsk::Cli::Main.start(["app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1"]) }.tap do |output|
-      assert_match "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", output
+      assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output
    end
  end

  private
-    def run_command(*command)
-      stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1"]) }
+    def run_command(*command, config: :with_accessories)
+      stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1"]) }
    end
 end
--- a/test/cli/build_test.rb
+++ b/test/cli/build_test.rb
@@ -9,6 +9,7 @@ class CliBuildTest < CliTestCase
  end

  test "push" do
+    Mrsk::Cli::Build.any_instance.stubs(:verify_local_dependencies).returns(true)
    run_command("push").tap do |output|
      assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder mrsk-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
    end
@@ -16,6 +17,7 @@ class CliBuildTest < CliTestCase

  test "push without builder" do
    stub_locking
+    Mrsk::Cli::Build.any_instance.stubs(:verify_local_dependencies).returns(true)
    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
      .with { |arg| arg == :docker }
      .raises(SSHKit::Command::Failed.new("no builder"))
@@ -30,7 +32,7 @@ class CliBuildTest < CliTestCase
  test "pull" do
    run_command("pull").tap do |output|
      assert_match /docker image rm --force dhh\/app:999/, output
-      assert_match /docker pull dhh\/app:latest/, output
+      assert_match /docker pull dhh\/app:999/, output
    end
  end

@@ -68,6 +70,23 @@ class CliBuildTest < CliTestCase
    end
  end

+  test "verify local dependencies" do
+    Mrsk::Commands::Builder.any_instance.stubs(:name).returns("remote".inquiry)
+
+    run_command("verify_local_dependencies").tap do |output|
+      assert_match /docker --version && docker buildx version/, output
+    end
+  end
+
+  test "verify local dependencies with no buildx plugin" do
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with(:docker, "--version", "&&", :docker, :buildx, "version")
+      .raises(SSHKit::Command::Failed.new("no buildx"))
+
+    Mrsk::Commands::Builder.any_instance.stubs(:native_and_local?).returns(false)
+    assert_raises(Mrsk::Cli::Build::BuildError) { run_command("verify_local_dependencies") }
+  end
+
  private
    def run_command(*command)
      stdouted { Mrsk::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
--- a/test/cli/cli_test_case.rb
+++ b/test/cli/cli_test_case.rb
@@ -1,5 +1,4 @@
 require "test_helper"
-require "active_support/testing/stream"

 class CliTestCase < ActiveSupport::TestCase
  include ActiveSupport::Testing::Stream
@@ -17,9 +16,4 @@ class CliTestCase < ActiveSupport::TestCase
    ENV.delete("MYSQL_ROOT_PASSWORD")
    ENV.delete("VERSION")
  end
-
-  private
-    def stdouted
-      capture(:stdout) { yield }.strip
-    end
 end
--- a/test/cli/healthcheck_test.rb
+++ b/test/cli/healthcheck_test.rb
@@ -5,62 +5,63 @@ class CliHealthcheckTest < CliTestCase
    # Prevent expected failures from outputting to terminal
    Thread.report_on_exception = false

-    SSHKit::Backend::Abstract.any_instance.stubs(:sleep) # No sleeping when retrying
+    Mrsk::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+
    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "MRSK_CONTAINER_NAME=\"healthcheck-app\"", "dhh/app:999")
+      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "MRSK_CONTAINER_NAME=\"healthcheck-app\"", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false)

    # Fail twice to test retry logic
    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with(:curl, "--silent", "--output", "/dev/null", "--write-out", "'%{http_code}'", "--max-time", "2", "http://localhost:3999/up")
-      .raises(SSHKit::Command::Failed)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("starting")
      .then
-      .raises(SSHKit::Command::Failed)
+      .returns("unhealthy")
      .then
-      .returns("200")
+      .returns("healthy")

    run_command("perform").tap do |output|
-      assert_match "Health check against /up failed to respond, retrying in 1s...", output
-      assert_match "Health check against /up failed to respond, retrying in 2s...", output
-      assert_match "Health check against /up succeeded with 200 OK!", output
+      assert_match "container not ready (starting), retrying in 1s (attempt 1/7)...", output
+      assert_match "container not ready (unhealthy), retrying in 2s (attempt 2/7)...", output
+      assert_match "Container is healthy!", output
    end
  end

-  test "perform failing because of curl" do
+  test "perform failing to become healthy" do
    # Prevent expected failures from outputting to terminal
    Thread.report_on_exception = false

-    SSHKit::Backend::Abstract.any_instance.stubs(:execute) # No need to execute anything here
+    Mrsk::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with(:docker, :run, "--detach", "--name", "healthcheck-app-999", "--publish", "3999:3000", "--label", "service=healthcheck-app", "-e", "MRSK_CONTAINER_NAME=\"healthcheck-app\"", "--health-cmd", "\"curl -f http://localhost:3000/up || exit 1\"", "--health-interval", "\"1s\"", "dhh/app:999")
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :container, :rm, raise_on_non_zero_exit: false)
+
+    # Continually report unhealthy
    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with(:curl, "--silent", "--output", "/dev/null", "--write-out", "'%{http_code}'", "--max-time", "2", "http://localhost:3999/up")
-      .returns("curl: command not found")
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("unhealthy")
+
+    # Capture logs when failing
    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :logs, "--tail", 50, "2>&1")
+      .returns("some log output")

-    exception = assert_raises SSHKit::Runner::ExecuteError do
-      run_command("perform")
-    end
-    assert_match "Health check against /up failed to return 200 OK!", exception.message
-  end
-
-  test "perform failing for unknown reason" do
-    # Prevent expected failures from outputting to terminal
-    Thread.report_on_exception = false
-
-    SSHKit::Backend::Abstract.any_instance.stubs(:execute) # No need to execute anything here
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with(:curl, "--silent", "--output", "/dev/null", "--write-out", "'%{http_code}'", "--max-time", "2", "http://localhost:3999/up")
-      .returns("500")
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :logs, "--tail", 50, "2>&1")
+    # Capture container health log when failing
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_pretty_json)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :inspect, "--format",  "'{{json .State.Health}}'")
+      .returns('{"Status":"unhealthy","Log":[{"ExitCode": 1,"Output": "/bin/sh: 1: curl: not found\n"}]}"')

    exception = assert_raises do
      run_command("perform")
    end
-    assert_match "Health check against /up failed with status 500", exception.message
+    assert_match "container not ready (unhealthy)", exception.message
  end

  private
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -12,20 +12,20 @@ class CliMainTest < CliTestCase
  test "deploy" do
    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }

-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:server:bootstrap", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)

    run_command("deploy").tap do |output|
-      assert_match /Ensure curl and Docker are installed/, output
      assert_match /Log into image registry/, output
      assert_match /Build and push app image/, output
      assert_match /Ensure Traefik is running/, output
      assert_match /Ensure app can pass healthcheck/, output
+      assert_match /Detect stale containers/, output
      assert_match /Prune old containers and images/, output
    end
  end
@@ -33,31 +33,90 @@ class CliMainTest < CliTestCase
  test "deploy with skip_push" do
    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }

-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:server:bootstrap", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)

    run_command("deploy", "--skip_push").tap do |output|
      assert_match /Acquiring the deploy lock/, output
-      assert_match /Ensure curl and Docker are installed/, output
      assert_match /Log into image registry/, output
      assert_match /Pull app image/, output
      assert_match /Ensure Traefik is running/, output
      assert_match /Ensure app can pass healthcheck/, output
+      assert_match /Detect stale containers/, output
      assert_match /Prune old containers and images/, output
      assert_match /Releasing the deploy lock/, output
    end
  end

+  test "deploy when locked" do
+    Thread.report_on_exception = false
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |*arg| arg[0..1] == [:mkdir, :mrsk_lock] }
+      .raises(RuntimeError, "mkdir: cannot create directory ‘mrsk_lock’: File exists")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:execute)
+      .with(:stat, :mrsk_lock, ">", "/dev/null", "&&", :cat, "mrsk_lock/details", "|", :base64, "-d")
+
+    assert_raises(Mrsk::Cli::LockError) do
+      run_command("deploy")
+    end
+  end
+
+  test "deploy error when locking" do
+    Thread.report_on_exception = false
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |*arg| arg[0..1] == [:mkdir, :mrsk_lock] }
+      .raises(SocketError, "getaddrinfo: nodename nor servname provided, or not known")
+
+    assert_raises(SSHKit::Runner::ExecuteError) do
+      run_command("deploy")
+    end
+  end
+
+  test "deploy errors during critical section leave lock in place" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }
+
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options).raises(RuntimeError)
+
+    assert !MRSK.holding_lock?
+    assert_raises(RuntimeError) do
+      stderred { run_command("deploy") }
+    end
+    assert MRSK.holding_lock?
+  end
+
+  test "deploy errors during outside section leave remove lock" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }
+
+    Mrsk::Cli::Main.any_instance.expects(:invoke)
+      .with("mrsk:cli:registry:login", [], invoke_options)
+      .raises(RuntimeError)
+
+    assert !MRSK.holding_lock?
+    assert_raises(RuntimeError) do
+      stderred { run_command("deploy") }
+    end
+    assert !MRSK.holding_lock?
+  end
+
  test "redeploy" do
    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" }

    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)

    run_command("redeploy").tap do |output|
@@ -71,6 +130,7 @@ class CliMainTest < CliTestCase

    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)

    run_command("redeploy", "--skip_push").tap do |output|
@@ -80,32 +140,46 @@ class CliMainTest < CliTestCase
  end

  test "rollback bad version" do
+    Thread.report_on_exception = false
+
    run_command("details") # Preheat MRSK const

    run_command("rollback", "nonsense").tap do |output|
-      assert_match /docker container ls --all --filter label=service=app --format '{{ .Names }}'/, output
+      assert_match /docker container ls --all --filter name=\^app-web-nonsense\$ --quiet/, output
      assert_match /The app version 'nonsense' is not available as a container/, output
    end
  end

  test "rollback good version" do
-    Mrsk::Cli::Main.any_instance.stubs(:container_name_available?).returns(true)
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1").returns("version-to-rollback\n").at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet")
+      .returns("version-to-rollback\n").at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-123$", "--quiet")
+      .returns("version-to-rollback\n").at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-")
+      .returns("version-to-rollback\n").at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=workers", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-")
+      .returns("version-to-rollback\n").at_least_once
+

    run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
      assert_match "Start version 123", output
-      assert_match "docker start app-123", output
-      assert_match "docker container ls --all --filter name=^app-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running"
+      assert_match "docker tag dhh/app:123 dhh/app:latest", output
+      assert_match "docker start app-web-123", output
+      assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running"
    end
  end

  test "rollback without old version" do
-    Mrsk::Cli::Main.any_instance.stubs(:container_name_available?).returns(true)
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1").returns("").at_least_once
+    Mrsk::Cli::Main.any_instance.stubs(:container_available?).returns(true)
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-").returns("").at_least_once

    run_command("rollback", "123").tap do |output|
      assert_match "Start version 123", output
-      assert_match "docker start app-123", output
+      assert_match "docker start app-web-123", output
      assert_no_match "docker stop", output
    end
  end
@@ -247,6 +321,19 @@ class CliMainTest < CliTestCase
    end
  end

+  test "broadcast" do
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with do |command, line, options, verbosity:|
+      command == "bin/audit_broadcast" &&
+        line =~ /\A'\[[^\]]+\] message'\z/ &&
+        options[:env].keys == %w[ MRSK_RECORDED_AT MRSK_PERFORMER MRSK_EVENT ] &&
+        verbosity == :debug
+    end.returns("Broadcast audit message: message")
+
+    run_command("broadcast", "-m", "message").tap do |output|
+      assert_match "Broadcast: message", output
+    end
+  end
+
  test "version" do
    version = stdouted { Mrsk::Cli::Main.new.version }
    assert_equal Mrsk::VERSION, version
--- a/test/cli/prune_test.rb
+++ b/test/cli/prune_test.rb
@@ -10,13 +10,13 @@ class CliPruneTest < CliTestCase

  test "images" do
    run_command("images").tap do |output|
-      assert_match /docker image prune --all --force --filter label=service=app --filter until=168h on 1.1.1.\d/, output
+      assert_match /docker image prune --force --filter label=service=app --filter dangling=true on 1.1.1.\d/, output
    end
  end

  test "containers" do
    run_command("containers").tap do |output|
-      assert_match /docker container prune --force --filter label=service=app --filter until=72h on 1.1.1.\d/, output
+      assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output
    end
  end

--- a/test/cli/server_test.rb
+++ b/test/cli/server_test.rb
@@ -1,11 +1,30 @@
 require_relative "cli_test_case"

 class CliServerTest < CliTestCase
-  test "bootstrap" do
+  test "bootstrap already installed" do
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(true).at_least_once
+
+    assert_equal "", run_command("bootstrap")
+  end
+
+  test "bootstrap install as non-root user" do
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ]', raise_on_non_zero_exit: false).returns(false).at_least_once
+
+    assert_raise RuntimeError, "Docker is not installed on 1.1.1.1, 1.1.1.3, 1.1.1.4, 1.1.1.2 and can't be automatically intalled without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/" do
+      run_command("bootstrap")
+    end
+  end
+
+  test "bootstrap install as root user" do
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ]', raise_on_non_zero_exit: false).returns(true).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:curl, "-fsSL", "https://get.docker.com", "|", :sh).at_least_once
+
    run_command("bootstrap").tap do |output|
-      assert_match /which curl/, output
-      assert_match /which docker/, output
-      assert_match /apt-get update -y && apt-get install curl docker.io -y/, output
+      ("1.1.1.1".."1.1.1.4").map do |host|
+        assert_match "Missing Docker on #{host}. Installing…", output
+      end
    end
  end

--- a/test/cli/traefik_test.rb
+++ b/test/cli/traefik_test.rb
@@ -3,7 +3,8 @@ require_relative "cli_test_case"
 class CliTraefikTest < CliTestCase
  test "boot" do
    run_command("boot").tap do |output|
-      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" traefik:v2.9.9 --providers.docker --log.level=DEBUG", output
+      assert_match "docker login", output
+      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{Mrsk::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=DEBUG", output
    end
  end

--- a/test/commander_test.rb
+++ b/test/commander_test.rb
@@ -2,9 +2,7 @@ require "test_helper"

 class CommanderTest < ActiveSupport::TestCase
  setup do
-    @mrsk = Mrsk::Commander.new.tap do |mrsk|
-      mrsk.configure config_file: Pathname.new(File.expand_path("fixtures/deploy_with_roles.yml", __dir__))
-    end
+    configure_with(:deploy_with_roles)
  end

  test "lazy configuration" do
@@ -47,12 +45,35 @@ class CommanderTest < ActiveSupport::TestCase
  end

  test "primary_host with specific hosts via role" do
-    @mrsk.specific_roles = "web"
-    assert_equal "1.1.1.1", @mrsk.primary_host
+    @mrsk.specific_roles = "workers"
+    assert_equal "1.1.1.3", @mrsk.primary_host
  end

  test "roles_on" do
    assert_equal [ "web" ], @mrsk.roles_on("1.1.1.1")
    assert_equal [ "workers" ], @mrsk.roles_on("1.1.1.3")
  end
+
+  test "default group strategy" do
+    assert_empty @mrsk.boot_strategy
+  end
+
+  test "specific limit group strategy" do
+    configure_with(:deploy_with_boot_strategy)
+
+    assert_equal({ in: :groups, limit: 3, wait: 2 }, @mrsk.boot_strategy)
+  end
+
+  test "percentage-based group strategy" do
+    configure_with(:deploy_with_precentage_boot_strategy)
+
+    assert_equal({ in: :groups, limit: 1, wait: 2 }, @mrsk.boot_strategy)
+  end
+
+  private
+    def configure_with(variant)
+      @mrsk = Mrsk::Commander.new.tap do |mrsk|
+        mrsk.configure config_file: Pathname.new(File.expand_path("fixtures/#{variant}.yml", __dir__))
+      end
+    end
 end
--- a/test/commands/app_test.rb
+++ b/test/commands/app_test.rb
@@ -13,7 +13,7 @@ class CommandsAppTest < ActiveSupport::TestCase

  test "run" do
    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

@@ -21,7 +21,7 @@ class CommandsAppTest < ActiveSupport::TestCase
    @config[:volumes] = ["/local/path:/container/path" ]

    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

@@ -29,7 +29,23 @@ class CommandsAppTest < ActiveSupport::TestCase
    @config[:healthcheck] = { "path" => "/healthz" }

    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/healthz\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/healthz || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      new_command.run.join(" ")
+  end
+
+  test "run with custom healthcheck command" do
+    @config[:healthcheck] = { "cmd" => "/bin/up" }
+
+    assert_equal \
+      "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"/bin/up\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      new_command.run.join(" ")
+  end
+
+  test "run with role-specific healthcheck options" do
+    @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } }
+
+    assert_equal \
+      "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"/bin/healthy\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

@@ -44,7 +60,7 @@ class CommandsAppTest < ActiveSupport::TestCase
    @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }

    assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
      new_command.run.join(" ")
  end

@@ -63,14 +79,14 @@ class CommandsAppTest < ActiveSupport::TestCase

  test "stop" do
    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop",
      new_command.stop.join(" ")
  end

  test "stop with custom stop wait time" do
    @config[:stop_wait_time] = 30
    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop -t 30",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop -t 30",
      new_command.stop.join(" ")
  end

@@ -96,37 +112,37 @@ class CommandsAppTest < ActiveSupport::TestCase

  test "logs" do
    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs 2>&1",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs 2>&1",
      new_command.logs.join(" ")

    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m 2>&1",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --since 5m 2>&1",
      new_command.logs(since: "5m").join(" ")

    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --tail 100 2>&1",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --tail 100 2>&1",
      new_command.logs(lines: "100").join(" ")

    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m --tail 100 2>&1",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --since 5m --tail 100 2>&1",
      new_command.logs(since: "5m", lines: "100").join(" ")

    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs 2>&1 | grep 'my-id'",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs 2>&1 | grep 'my-id'",
      new_command.logs(grep: "my-id").join(" ")

    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
      new_command.logs(since: "5m", grep: "my-id").join(" ")
  end

  test "follow logs" do
    assert_match \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1",
      new_command.follow_logs(host: "app-1")

    assert_match \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1 | grep \"Completed\"",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1 | grep \"Completed\"",
      new_command.follow_logs(host: "app-1", grep: "Completed")
  end

@@ -178,17 +194,17 @@ class CommandsAppTest < ActiveSupport::TestCase
  end


-  test "current_container_id" do
+  test "current_running_container_id" do
    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web",
-      new_command.current_container_id.join(" ")
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest",
+      new_command.current_running_container_id.join(" ")
  end

-  test "current_container_id with destination" do
+  test "current_running_container_id with destination" do
    @destination = "staging"
    assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web",
-      new_command.current_container_id.join(" ")
+      "docker ps --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --latest",
+      new_command.current_running_container_id.join(" ")
  end

  test "container_id_for" do
@@ -199,10 +215,20 @@ class CommandsAppTest < ActiveSupport::TestCase

  test "current_running_version" do
    assert_equal \
-      "docker ps --filter label=service=app --filter label=role=web --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1",
+      "docker ps --filter label=service=app --filter label=role=web --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-",
      new_command.current_running_version.join(" ")
  end

+  test "list_versions" do
+    assert_equal \
+      "docker ps --filter label=service=app --filter label=role=web --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-",
+      new_command.list_versions.join(" ")
+
+    assert_equal \
+      "docker ps --filter label=service=app --filter label=role=web --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-",
+      new_command.list_versions("--latest", status: :running).join(" ")
+  end
+
  test "list_containers" do
    assert_equal \
      "docker container ls --all --filter label=service=app --filter label=role=web",
@@ -267,6 +293,12 @@ class CommandsAppTest < ActiveSupport::TestCase
      new_command.remove_images.join(" ")
  end

+  test "tag_current_as_latest" do
+    assert_equal \
+      "docker tag dhh/app:999 dhh/app:latest",
+      new_command.tag_current_as_latest.join(" ")
+  end
+
  private
    def new_command(role: "web")
      Mrsk::Commands::App.new(Mrsk::Configuration.new(@config, destination: @destination, version: "999"), role: role)
--- a/test/commands/auditor_test.rb
+++ b/test/commands/auditor_test.rb
@@ -6,38 +6,65 @@ class CommandsAuditorTest < ActiveSupport::TestCase
      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
      audit_broadcast_cmd: "bin/audit_broadcast"
    }
+
+    @auditor = new_command
  end

  test "record" do
-    assert_match \
-      /echo '.* app removed container' >> mrsk-app-audit.log/,
-      new_command.record("app removed container").join(" ")
+    assert_equal [
+      :echo,
+      "[#{@auditor.details[:recorded_at]}]", "[#{@auditor.details[:performer]}]",
+      "app removed container",
+      ">>", "mrsk-app-audit.log"
+    ], @auditor.record("app removed container")
  end

  test "record with destination" do
-    @destination = "staging"
-
-    assert_match \
-      /echo '.* app removed container' >> mrsk-app-staging-audit.log/,
-      new_command.record("app removed container").join(" ")
+    new_command(destination: "staging").tap do |auditor|
+      assert_equal [
+        :echo,
+        "[#{auditor.details[:recorded_at]}]", "[#{auditor.details[:performer]}]", "[#{auditor.details[:destination]}]",
+        "app removed container",
+        ">>", "mrsk-app-staging-audit.log"
+      ], auditor.record("app removed container")
+    end
  end

-  test "record with role" do
-    @role = "web"
+  test "record with command details" do
+    new_command(role: "web").tap do |auditor|
+      assert_equal [
+        :echo,
+        "[#{auditor.details[:recorded_at]}]", "[#{auditor.details[:performer]}]", "[#{auditor.details[:role]}]",
+        "app removed container",
+        ">>", "mrsk-app-audit.log"
+      ], auditor.record("app removed container")
+    end
+  end

-    assert_match \
-      /echo '.* \[web\] app removed container' >> mrsk-app-audit.log/,
-      new_command.record("app removed container").join(" ")
+  test "record with arg details" do
+    assert_equal [
+      :echo,
+      "[#{@auditor.details[:recorded_at]}]", "[#{@auditor.details[:performer]}]", "[value]",
+      "app removed container",
+      ">>", "mrsk-app-audit.log"
+    ], @auditor.record("app removed container", detail: "value")
  end

  test "broadcast" do
-    assert_match \
-      /bin\/audit_broadcast '\[.*\] app removed container'/,
-      new_command.broadcast("app removed container").join(" ")
+    assert_equal [
+      "bin/audit_broadcast",
+      "'[#{@auditor.details[:performer]}] [value] app removed container'",
+      env: {
+        "MRSK_RECORDED_AT" => @auditor.details[:recorded_at],
+        "MRSK_PERFORMER" => @auditor.details[:performer],
+        "MRSK_EVENT" => "app removed container",
+        "MRSK_DETAIL" => "value"
+      }
+    ], @auditor.broadcast("app removed container", detail: "value")
  end

  private
-    def new_command
-      Mrsk::Commands::Auditor.new(Mrsk::Configuration.new(@config, destination: @destination, version: "123"), role: @role)
+    def new_command(destination: nil, **details)
+      Mrsk::Commands::Auditor.new(Mrsk::Configuration.new(@config, destination: destination, version: "123"), **details)
    end
 end
--- a/test/commands/builder_test.rb
+++ b/test/commands/builder_test.rb
@@ -52,12 +52,21 @@ class CommandsBuilderTest < ActiveSupport::TestCase
  end

  test "build dockerfile" do
+    Pathname.any_instance.expects(:exist?).returns(true).once
    builder = new_builder_command(builder: { "dockerfile" => "Dockerfile.xyz" })
    assert_equal \
      "-t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile.xyz",
      builder.target.build_options.join(" ")
  end

+  test "missing dockerfile" do
+    Pathname.any_instance.expects(:exist?).returns(false).once
+    builder = new_builder_command(builder: { "dockerfile" => "Dockerfile.xyz" })
+    assert_raises(Mrsk::Commands::Builder::Base::BuilderError) do
+      builder.target.build_options.join(" ")
+    end
+  end
+
  test "build context" do
    builder = new_builder_command(builder: { "context" => ".." })
    assert_equal \
--- a/test/commands/docker_test.rb
+++ b/test/commands/docker_test.rb
@@ -0,0 +1,26 @@
+require "test_helper"
+
+class CommandsDockerTest < ActiveSupport::TestCase
+  setup do
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ]
+    }
+    @docker = Mrsk::Commands::Docker.new(Mrsk::Configuration.new(@config))
+  end
+
+  test "install" do
+    assert_equal "curl -fsSL https://get.docker.com | sh", @docker.install.join(" ")
+  end
+
+  test "installed?" do
+    assert_equal "docker -v", @docker.installed?.join(" ")
+  end
+
+  test "running?" do
+    assert_equal "docker version", @docker.running?.join(" ")
+  end
+
+  test "superuser?" do
+    assert_equal '[ "${EUID:-$(id -u)}" -eq 0 ]', @docker.superuser?.join(" ")
+  end
+end
--- a/test/commands/healthcheck_test.rb
+++ b/test/commands/healthcheck_test.rb
@@ -10,7 +10,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase

  test "run" do
    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
      new_command.run.join(" ")
  end

@@ -18,7 +18,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
    @config[:healthcheck] = { "port" => 3001 }

    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3001 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 3999:3001 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"curl -f http://localhost:3001/up || exit 1\" --health-interval \"1s\" dhh/app:123",
      new_command.run.join(" ")
  end

@@ -26,29 +26,35 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
    @destination = "staging"

    assert_equal \
-      "docker run --detach --name healthcheck-app-staging-123 --publish 3999:3000 --label service=healthcheck-app-staging -e MRSK_CONTAINER_NAME=\"healthcheck-app-staging\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-staging-123 --publish 3999:3000 --label service=healthcheck-app-staging -e MRSK_CONTAINER_NAME=\"healthcheck-app-staging\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
+      new_command.run.join(" ")
+  end
+
+  test "run with custom healthcheck" do
+    @config[:healthcheck] = { "cmd" => "/bin/up" }
+
+    assert_equal \
+      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"/bin/up\" --health-interval \"1s\" dhh/app:123",
      new_command.run.join(" ")
  end

  test "run with custom options" do
    @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere" } } }
    assert_equal \
-      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" --mount \"somewhere\" dhh/app:123",
+      "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --mount \"somewhere\" dhh/app:123",
      new_command.run.join(" ")
  end

-  test "curl" do
+  test "status" do
    assert_equal \
-      "curl --silent --output /dev/null --write-out '%{http_code}' --max-time 2 http://localhost:3999/up",
-      new_command.curl.join(" ")
+      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'",
+      new_command.status.join(" ")
  end

-  test "curl with custom path" do
-    @config[:healthcheck] = { "path" => "/healthz" }
-
+  test "container_health_log" do
    assert_equal \
-      "curl --silent --output /dev/null --write-out '%{http_code}' --max-time 2 http://localhost:3999/healthz",
-      new_command.curl.join(" ")
+      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker inspect --format '{{json .State.Health}}'",
+      new_command.container_health_log.join(" ")
  end

  test "stop" do
--- a/test/commands/prune_test.rb
+++ b/test/commands/prune_test.rb
@@ -10,13 +10,13 @@ class CommandsPruneTest < ActiveSupport::TestCase

  test "images" do
    assert_equal \
-      "docker image prune --all --force --filter label=service=app --filter until=168h",
+      "docker image prune --force --filter label=service=app --filter dangling=true",
      new_command.images.join(" ")
  end

  test "containers" do
    assert_equal \
-      "docker container prune --force --filter label=service=app --filter until=72h",
+      "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done",
      new_command.containers.join(" ")
  end

--- a/test/commands/traefik_test.rb
+++ b/test/commands/traefik_test.rb
@@ -2,53 +2,66 @@ require "test_helper"

 class CommandsTraefikTest < ActiveSupport::TestCase
  setup do
+    @image = "traefik:test"
+
    @config = {
      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
-      traefik: { "args" => { "accesslog.format" => "json", "api.insecure" => true, "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
+      traefik: { "image" => @image, "args" => { "accesslog.format" => "json", "api.insecure" => true, "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
    }
  end

  test "run" do
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" traefik:v2.9.9 --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")

    @config[:traefik]["host_port"] = "8080"
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" traefik:v2.9.9 --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")
  end

  test "run with ports configured" do
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" traefik:v2.9.9 --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")

    @config[:traefik]["options"] = {"publish" => %w[9000:9000 9001:9001]}
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --publish \"9000:9000\" --publish \"9001:9001\" traefik:v2.9.9 --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --publish \"9000:9000\" --publish \"9001:9001\" #{@image} --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")
  end

  test "run with volumes configured" do
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" traefik:v2.9.9 --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")

    @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json] }
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" traefik:v2.9.9 --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" #{@image} --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")
  end

  test "run with several options configured" do
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" traefik:v2.9.9 --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")

    @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json], "publish" => %w[8080:8080], "memory" => "512m"}
    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" traefik:v2.9.9 --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" #{@image} --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      new_command.run.join(" ")
+  end
+
+  test "run with labels configured" do
+    assert_equal \
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{@image} --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      new_command.run.join(" ")
+
+    @config[:traefik]["labels"] = { "traefik.http.routers.dashboard.service" => "api@internal", "traefik.http.routers.dashboard.middlewares" => "auth" }
+    assert_equal \
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --label traefik.http.routers.dashboard.service=\"api@internal\" --label traefik.http.routers.dashboard.middlewares=\"auth\" #{@image} --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")
  end

@@ -56,7 +69,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
    @config.delete(:traefik)

    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" traefik:v2.9.9 --providers.docker --log.level=DEBUG",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{Mrsk::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=DEBUG",
      new_command.run.join(" ")
  end

@@ -64,7 +77,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
    @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }

    assert_equal \
-      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" traefik:v2.9.9 --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
+      "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" #{@image} --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"",
      new_command.run.join(" ")
  end

--- a/test/configuration/accessory_test.rb
+++ b/test/configuration/accessory_test.rb
@@ -112,8 +112,11 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase

  test "env args with secret" do
    ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
-    assert_equal ["-e", "MYSQL_ROOT_PASSWORD=\"secret123\"", "-e", "MYSQL_ROOT_HOST=\"%\""], @config.accessory(:mysql).env_args
-    assert @config.accessory(:mysql).env_args[1].is_a?(SSHKit::Redaction)
+
+    @config.accessory(:mysql).env_args.tap do |env_args|
+      assert_equal ["-e", "MYSQL_ROOT_PASSWORD=\"secret123\"", "-e", "MYSQL_ROOT_HOST=\"%\""], Mrsk::Utils.unredacted(env_args)
+      assert_equal ["-e", "MYSQL_ROOT_PASSWORD=[REDACTED]", "-e", "MYSQL_ROOT_HOST=\"%\""], Mrsk::Utils.redacted(env_args)
+    end
  ensure
    ENV["MYSQL_ROOT_PASSWORD"] = nil
  end
--- a/test/configuration/role_test.rb
+++ b/test/configuration/role_test.rb
@@ -42,7 +42,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
  end

  test "special label args for web" do
-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\"", "--label", "traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\"", "--label", "traefik.http.middlewares.app-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app.middlewares=\"app-retry@docker\"" ], @config.role(:web).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args
  end

  test "custom labels" do
@@ -57,8 +57,8 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
  end

  test "overwriting default traefik label" do
-    @deploy[:labels] = { "traefik.http.routers.app.rule" => "\"Host(\\`example.com\\`) || (Host(\\`example.org\\`) && Path(\\`/traefik\\`))\"" }
-    assert_equal "\"Host(\\`example.com\\`) || (Host(\\`example.org\\`) && Path(\\`/traefik\\`))\"", @config.role(:web).labels["traefik.http.routers.app.rule"]
+    @deploy[:labels] = { "traefik.http.routers.app-web.rule" => "\"Host(\\`example.com\\`) || (Host(\\`example.org\\`) && Path(\\`/traefik\\`))\"" }
+    assert_equal "\"Host(\\`example.com\\`) || (Host(\\`example.org\\`) && Path(\\`/traefik\\`))\"", @config.role(:web).labels["traefik.http.routers.app-web.rule"]
  end

  test "default traefik label on non-web role" do
@@ -66,7 +66,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
      c[:servers]["beta"] = { "traefik" => "true", "hosts" => [ "1.1.1.5" ] }
    })

-    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\"", "--label", "traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\"", "--label", "traefik.http.middlewares.app-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app.middlewares=\"app-retry@docker\"" ], config.role(:beta).label_args
+    assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "traefik.http.services.app-beta.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args
  end

  test "env overwritten by role" do
@@ -97,7 +97,10 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
    ENV["REDIS_PASSWORD"] = "secret456"
    ENV["DB_PASSWORD"] = "secret&\"123"

-    assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "DB_PASSWORD=\"secret&\\\"123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], @config_with_roles.role(:workers).env_args
+    @config_with_roles.role(:workers).env_args.tap do |env_args|
+      assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "DB_PASSWORD=\"secret&\\\"123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.unredacted(env_args)
+      assert_equal ["-e", "REDIS_PASSWORD=[REDACTED]", "-e", "DB_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.redacted(env_args)
+    end
  ensure
    ENV["REDIS_PASSWORD"] = nil
    ENV["DB_PASSWORD"] = nil
@@ -116,7 +119,10 @@ class ConfigurationRoleTest < ActiveSupport::TestCase

    ENV["DB_PASSWORD"] = "secret123"

-    assert_equal ["-e", "DB_PASSWORD=\"secret123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], @config_with_roles.role(:workers).env_args
+    @config_with_roles.role(:workers).env_args.tap do |env_args|
+      assert_equal ["-e", "DB_PASSWORD=\"secret123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.unredacted(env_args)
+      assert_equal ["-e", "DB_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.redacted(env_args)
+    end
  ensure
    ENV["DB_PASSWORD"] = nil
  end
@@ -133,7 +139,10 @@ class ConfigurationRoleTest < ActiveSupport::TestCase

    ENV["REDIS_PASSWORD"] = "secret456"

-    assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], @config_with_roles.role(:workers).env_args
+    @config_with_roles.role(:workers).env_args.tap do |env_args|
+      assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.unredacted(env_args)
+      assert_equal ["-e", "REDIS_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.redacted(env_args)
+    end
  ensure
    ENV["REDIS_PASSWORD"] = nil
  end
--- a/test/configuration_test.rb
+++ b/test/configuration_test.rb
@@ -48,7 +48,7 @@ class ConfigurationTest < ActiveSupport::TestCase
  end

  test "role" do
-    assert_equal "web", @config.role(:web).name
+    assert @config.role(:web).name.web?
    assert_equal "workers", @config_with_roles.role(:workers).name
    assert_nil @config.role(:missing)
  end
@@ -113,12 +113,13 @@ class ConfigurationTest < ActiveSupport::TestCase

  test "env args with clear and secrets" do
    ENV["PASSWORD"] = "secret123"
+
    config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
      env: { "clear" => { "PORT" => "3000" }, "secret" => [ "PASSWORD" ] }
    }) })

-    assert_equal [ "-e", "PASSWORD=\"secret123\"", "-e", "PORT=\"3000\"" ], config.env_args
-    assert config.env_args[1].is_a?(SSHKit::Redaction)
+    assert_equal [ "-e", "PASSWORD=\"secret123\"", "-e", "PORT=\"3000\"" ], Mrsk::Utils.unredacted(config.env_args)
+    assert_equal [ "-e", "PASSWORD=[REDACTED]", "-e", "PORT=\"3000\"" ], Mrsk::Utils.redacted(config.env_args)
  ensure
    ENV["PASSWORD"] = nil
  end
@@ -133,12 +134,13 @@ class ConfigurationTest < ActiveSupport::TestCase

  test "env args with only secrets" do
    ENV["PASSWORD"] = "secret123"
+
    config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
      env: { "secret" => [ "PASSWORD" ] }
    }) })

-    assert_equal [ "-e", "PASSWORD=\"secret123\"" ], config.env_args
-    assert config.env_args[1].is_a?(SSHKit::Redaction)
+    assert_equal [ "-e", "PASSWORD=\"secret123\"" ], Mrsk::Utils.unredacted(config.env_args)
+    assert_equal [ "-e", "PASSWORD=[REDACTED]" ], Mrsk::Utils.redacted(config.env_args)
  ensure
    ENV["PASSWORD"] = nil
  end
@@ -247,6 +249,6 @@ class ConfigurationTest < ActiveSupport::TestCase
  end

  test "to_h" do
-    assert_equal({ :roles=>["web"], :hosts=>["1.1.1.1", "1.1.1.2"], :primary_host=>"1.1.1.1", :version=>"missing", :repository=>"dhh/app", :absolute_image=>"dhh/app:missing", :service_with_version=>"app-missing", :env_args=>["-e", "REDIS_URL=\"redis://x/y\""], :ssh_options=>{:user=>"root", :auth_methods=>["publickey"]}, :volume_args=>["--volume", "/local/path:/container/path"], :logging=>["--log-opt", "max-size=\"10m\""], :healthcheck=>{"path"=>"/up", "port"=>3000 }}, @config.to_h)
+    assert_equal({ :roles=>["web"], :hosts=>["1.1.1.1", "1.1.1.2"], :primary_host=>"1.1.1.1", :version=>"missing", :repository=>"dhh/app", :absolute_image=>"dhh/app:missing", :service_with_version=>"app-missing", :env_args=>["-e", "REDIS_URL=\"redis://x/y\""], :ssh_options=>{:user=>"root", :auth_methods=>["publickey"]}, :volume_args=>["--volume", "/local/path:/container/path"], :logging=>["--log-opt", "max-size=\"10m\""], :healthcheck=>{"path"=>"/up", "port"=>3000, "max_attempts" => 7 }}, @config.to_h)
  end
 end
--- a/test/fixtures/deploy_simple.yml
+++ b/test/fixtures/deploy_simple.yml
@@ -6,3 +6,4 @@ servers:
 registry:
  username: user
  password: pw
+audit_broadcast_cmd: "bin/audit_broadcast"
--- a/test/fixtures/deploy_with_boot_strategy.yml
+++ b/test/fixtures/deploy_with_boot_strategy.yml
@@ -0,0 +1,17 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+
+registry:
+  username: user
+  password: pw
+
+boot:
+  limit: 3
+  wait: 2
--- a/test/fixtures/deploy_with_precentage_boot_strategy.yml
+++ b/test/fixtures/deploy_with_precentage_boot_strategy.yml
@@ -0,0 +1,17 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+
+registry:
+  username: user
+  password: pw
+
+boot:
+  limit: 25%
+  wait: 2
--- a/test/integration/deploy_test.rb
+++ b/test/integration/deploy_test.rb
@@ -0,0 +1,132 @@
+require "net/http"
+require "test_helper"
+
+class DeployTest < ActiveSupport::TestCase
+
+  setup do
+    docker_compose "up --build --force-recreate -d"
+    wait_for_healthy
+  end
+
+  teardown do
+    docker_compose "down -v"
+  end
+
+  test "deploy" do
+    first_version = latest_app_version
+
+    assert_app_is_down
+
+    mrsk :deploy
+
+    assert_app_is_up version: first_version
+
+    second_version = update_app_rev
+
+    mrsk :redeploy
+
+    assert_app_is_up version: second_version
+
+    mrsk :rollback, first_version
+
+    assert_app_is_up version: first_version
+
+    details = mrsk :details, capture: true
+
+    assert_match /Traefik Host: vm1/, details
+    assert_match /Traefik Host: vm2/, details
+    assert_match /App Host: vm1/, details
+    assert_match /App Host: vm2/, details
+    assert_match /traefik:v2.9/, details
+    assert_match /registry:4443\/app:#{first_version}/, details
+
+    audit = mrsk :audit, capture: true
+
+    assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
+  end
+
+  private
+    def docker_compose(*commands, capture: false)
+      command = "docker compose #{commands.join(" ")}"
+      succeeded = false
+      if capture
+        result = stdouted { succeeded = system("cd test/integration && #{command}") }
+      else
+        succeeded = system("cd test/integration && #{command}")
+      end
+
+      raise "Command `#{command}` failed with error code `#{$?}`" unless succeeded
+      result
+    end
+
+    def deployer_exec(*commands, **options)
+      docker_compose("exec deployer #{commands.join(" ")}", **options)
+    end
+
+    def mrsk(*commands, **options)
+      deployer_exec(:mrsk, *commands, **options)
+    end
+
+    def assert_app_is_down
+      assert_equal "502", app_response.code
+    end
+
+    def assert_app_is_up(version: nil)
+      code = app_response.code
+      if code != "200"
+        puts "Got response code #{code}, here are the traefik logs:"
+        mrsk :traefik, :logs
+        puts "And here are the load balancer logs"
+        docker_compose :logs, :load_balancer
+        puts "Tried to get the response code again and got #{app_response.code}"
+      end
+      assert_equal "200", code
+      assert_app_version(version) if version
+    end
+
+    def assert_app_not_found
+      assert_equal "404", app_response.code
+    end
+
+    def wait_for_app_to_be_up(timeout: 10, up_count: 3)
+      timeout_at = Time.now + timeout
+      up_times = 0
+      response = app_response
+      while up_times < up_count && timeout_at > Time.now
+        sleep 0.1
+        up_times += 1 if response.code == "200"
+        response = app_response
+      end
+      assert_equal up_times, up_count
+    end
+
+    def app_response
+      Net::HTTP.get_response(URI.parse("http://localhost:12345"))
+    end
+
+    def update_app_rev
+      deployer_exec "./update_app_rev.sh"
+      latest_app_version
+    end
+
+    def latest_app_version
+      deployer_exec("cat version", capture: true)
+    end
+
+    def assert_app_version(version)
+      actual_version = Net::HTTP.get_response(URI.parse("http://localhost:12345/version")).body.strip
+
+      assert_equal version, actual_version
+    end
+
+    def wait_for_healthy(timeout: 20)
+      timeout_at = Time.now + timeout
+      while docker_compose("ps -a | tail -n +2 | grep -v '(healthy)' | wc -l", capture: true) != "0"
+        if timeout_at < Time.now
+          docker_compose("ps -a | tail -n +2 | grep -v '(healthy)'")
+          raise "Container not healthy after #{timeout} seconds" if timeout_at < Time.now
+        end
+        sleep 0.1
+      end
+    end
+end
--- a/test/integration/docker-compose.yml
+++ b/test/integration/docker-compose.yml
@@ -0,0 +1,50 @@
+version: "3.7"
+name: "mrsk-test"
+
+volumes:
+  shared:
+
+services:
+  shared:
+    build:
+      context: docker/shared
+    volumes:
+      - shared:/shared
+
+  deployer:
+    privileged: true
+    build:
+      context: docker/deployer
+    volumes:
+      - ../..:/mrsk
+      - shared:/shared
+
+  registry:
+    build:
+      context: docker/registry
+    environment:
+      - REGISTRY_HTTP_ADDR=0.0.0.0:4443
+      - REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt
+      - REGISTRY_HTTP_TLS_KEY=/certs/domain.key
+    volumes:
+      - shared:/shared
+
+  vm1:
+    privileged: true
+    build:
+      context: docker/vm
+    volumes:
+      - shared:/shared
+
+  vm2:
+    privileged: true
+    build:
+      context: docker/vm
+    volumes:
+      - shared:/shared
+
+  load_balancer:
+    build:
+      context: docker/load_balancer
+    ports:
+      - "12345:80"
--- a/test/integration/docker/deployer/Dockerfile
+++ b/test/integration/docker/deployer/Dockerfile
@@ -0,0 +1,30 @@
+FROM ruby:3.2
+
+WORKDIR /app
+
+RUN apt-get update --fix-missing && apt-get install -y ca-certificates openssh-client curl gnupg docker.io
+
+RUN install -m 0755 -d /etc/apt/keyrings
+RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+RUN chmod a+r /etc/apt/keyrings/docker.gpg
+RUN echo \
+  "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+  "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
+  tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+RUN apt-get update --fix-missing && apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+
+COPY *.sh .
+COPY app/ .
+
+RUN ln -s /shared/ssh /root/.ssh
+RUN mkdir -p /etc/docker/certs.d/registry:4443 && ln -s /shared/certs/domain.crt /etc/docker/certs.d/registry:4443/ca.crt
+
+RUN git config --global user.email "deployer@example.com"
+RUN git config --global user.name "Deployer"
+RUN git init && git add . && git commit -am "Initial version"
+RUN git rev-parse HEAD > version
+
+HEALTHCHECK --interval=1s CMD pgrep sleep
+
+CMD ["./boot.sh"]
--- a/test/integration/docker/deployer/app/Dockerfile
+++ b/test/integration/docker/deployer/app/Dockerfile
@@ -0,0 +1,4 @@
+FROM nginx:1-alpine-slim
+
+COPY default.conf /etc/nginx/conf.d/default.conf
+COPY version /usr/share/nginx/html/version
--- a/test/integration/docker/deployer/app/config/deploy.yml
+++ b/test/integration/docker/deployer/app/config/deploy.yml
@@ -0,0 +1,17 @@
+service: app
+image: app
+servers:
+  - vm1
+  - vm2
+registry:
+  server: registry:4443
+  username: root
+  password: root
+builder:
+  multiarch: false
+healthcheck:
+  cmd: wget -qO- http://localhost > /dev/null
+traefik:
+  args:
+    accesslog: true
+    accesslog.format: json
--- a/test/integration/docker/deployer/app/default.conf
+++ b/test/integration/docker/deployer/app/default.conf
@@ -0,0 +1,17 @@
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  localhost;
+
+    location / {
+        root   /usr/share/nginx/html;
+        index  index.html index.htm;
+    }
+
+    # redirect server error pages to the static page /50x.html
+    #
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+}
--- a/test/integration/docker/deployer/boot.sh
+++ b/test/integration/docker/deployer/boot.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+cd /mrsk && gem build mrsk.gemspec -o /tmp/mrsk.gem && gem install /tmp/mrsk.gem
+
+dockerd &
+
+trap "pkill -f sleep" term
+
+sleep infinity & wait
--- a/test/integration/docker/deployer/update_app_rev.sh
+++ b/test/integration/docker/deployer/update_app_rev.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+git commit -am 'Update rev' --amend
+git rev-parse HEAD > version
--- a/test/integration/docker/load_balancer/Dockerfile
+++ b/test/integration/docker/load_balancer/Dockerfile
@@ -0,0 +1,5 @@
+FROM nginx:1-alpine-slim
+
+COPY default.conf /etc/nginx/conf.d/default.conf
+
+HEALTHCHECK --interval=1s CMD pgrep nginx
--- a/test/integration/docker/load_balancer/default.conf
+++ b/test/integration/docker/load_balancer/default.conf
@@ -0,0 +1,12 @@
+upstream loadbalancer {
+  server vm1:80;
+  server vm2:80;
+}
+
+server {
+  listen 80;
+
+  location / {
+    proxy_pass http://loadbalancer;
+  }
+}
--- a/test/integration/docker/registry/Dockerfile
+++ b/test/integration/docker/registry/Dockerfile
@@ -0,0 +1,9 @@
+FROM registry
+
+COPY boot.sh .
+
+RUN ln -s /shared/certs /certs
+
+HEALTHCHECK --interval=1s CMD pgrep registry
+
+ENTRYPOINT ["./boot.sh"]
--- a/test/integration/docker/registry/boot.sh
+++ b/test/integration/docker/registry/boot.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+while [ ! -f /certs/domain.crt ]; do sleep 1; done
+
+trap "pkill -f registry" term
+
+/entrypoint.sh /etc/docker/registry/config.yml & wait
--- a/test/integration/docker/shared/Dockerfile
+++ b/test/integration/docker/shared/Dockerfile
@@ -0,0 +1,17 @@
+FROM ubuntu:22.10
+
+WORKDIR /work
+
+RUN apt-get update --fix-missing && apt-get -y install openssh-client openssl
+
+RUN mkdir ssh && \
+  ssh-keygen -t rsa -f ssh/id_rsa -N ""
+
+COPY registry-dns.conf .
+COPY boot.sh .
+
+RUN mkdir certs && openssl req -newkey rsa:4096 -nodes -sha256 -keyout certs/domain.key   -x509 -days 365 -out certs/domain.crt   -subj '/CN=registry' -extensions EXT -config registry-dns.conf
+
+HEALTHCHECK --interval=1s CMD pgrep sleep
+
+CMD ["./boot.sh"]
--- a/test/integration/docker/shared/boot.sh
+++ b/test/integration/docker/shared/boot.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cp -r * /shared
+
+trap "pkill -f sleep" term
+
+sleep infinity & wait
--- a/test/integration/docker/shared/registry-dns.conf
+++ b/test/integration/docker/shared/registry-dns.conf
@@ -0,0 +1,7 @@
+[dn]
+CN=registry
+[req]
+distinguished_name = dn
+[EXT]
+subjectAltName=DNS:registry
+keyUsage=digitalSignature
--- a/test/integration/docker/vm/Dockerfile
+++ b/test/integration/docker/vm/Dockerfile
@@ -0,0 +1,14 @@
+FROM ubuntu:22.10
+
+WORKDIR /work
+
+RUN apt-get update --fix-missing && apt-get -y install openssh-client openssh-server docker.io
+
+RUN mkdir /root/.ssh && ln -s /shared/ssh/id_rsa.pub /root/.ssh/authorized_keys
+RUN mkdir -p /etc/docker/certs.d/registry:4443 && ln -s /shared/certs/domain.crt /etc/docker/certs.d/registry:4443/ca.crt
+
+COPY boot.sh .
+
+HEALTHCHECK --interval=1s CMD pgrep dockerd
+
+CMD ["./boot.sh"]
--- a/test/integration/docker/vm/boot.sh
+++ b/test/integration/docker/vm/boot.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+while [ ! -f /root/.ssh/authorized_keys ]; do echo "Waiting for ssh keys"; sleep 1; done
+
+service ssh restart
+
+dockerd &
+
+trap "pkill -f sleep" term
+
+sleep infinity & wait
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -1,6 +1,7 @@
 require "bundler/setup"
 require "active_support/test_case"
 require "active_support/testing/autorun"
+require "active_support/testing/stream"
 require "debug"
 require "mocha/minitest" # using #stubs that can alter returns
 require "minitest/autorun" # using #stub that take args
@@ -23,4 +24,14 @@ module SSHKit
 end

 class ActiveSupport::TestCase
+  include ActiveSupport::Testing::Stream
+
+  private
+    def stdouted
+      capture(:stdout) { yield }.strip
+    end
+
+    def stderred
+      capture(:stderr) { yield }.strip
+    end
 end
--- a/test/utils_test.rb
+++ b/test/utils_test.rb
@@ -8,13 +8,16 @@ class UtilsTest < ActiveSupport::TestCase

  test "argumentize with redacted" do
    assert_kind_of SSHKit::Redaction, \
-      Mrsk::Utils.argumentize("--label", { foo: "bar" }, redacted: true).last
+      Mrsk::Utils.argumentize("--label", { foo: "bar" }, sensitive: true).last
  end

  test "argumentize_env_with_secrets" do
    ENV.expects(:fetch).with("FOO").returns("secret")
-    assert_equal [ "-e", "FOO=\"secret\"", "-e", "BAZ=\"qux\"" ], \
-      Mrsk::Utils.argumentize_env_with_secrets({ "secret" => [ "FOO" ], "clear" => { BAZ: "qux" } })
+
+    args = Mrsk::Utils.argumentize_env_with_secrets({ "secret" => [ "FOO" ], "clear" => { BAZ: "qux" } })
+
+    assert_equal [ "-e", "FOO=[REDACTED]", "-e", "BAZ=\"qux\"" ], Mrsk::Utils.redacted(args)
+    assert_equal [ "-e", "FOO=\"secret\"", "-e", "BAZ=\"qux\"" ], Mrsk::Utils.unredacted(args)
  end

  test "optionize" do
@@ -27,13 +30,35 @@ class UtilsTest < ActiveSupport::TestCase
      Mrsk::Utils.optionize({ foo: "bar", baz: "qux", quux: true }, with: "=")
  end

-  test "redact" do
-    assert_kind_of SSHKit::Redaction, Mrsk::Utils.redact("secret")
-    assert_equal "secret", Mrsk::Utils.redact("secret")
+  test "no redaction from #to_s" do
+    assert_equal "secret", Mrsk::Utils.sensitive("secret").to_s
+  end
+
+  test "redact from #inspect" do
+    assert_equal "[REDACTED]".inspect, Mrsk::Utils.sensitive("secret").inspect
+  end
+
+  test "redact from SSHKit output" do
+    assert_kind_of SSHKit::Redaction, Mrsk::Utils.sensitive("secret")
+  end
+
+  test "redact from YAML output" do
+    assert_equal "--- ! '[REDACTED]'\n", YAML.dump(Mrsk::Utils.sensitive("secret"))
  end

  test "escape_shell_value" do
    assert_equal "\"foo\"", Mrsk::Utils.escape_shell_value("foo")
    assert_equal "\"\\`foo\\`\"", Mrsk::Utils.escape_shell_value("`foo`")
+
+    assert_equal "\"${PWD}\"", Mrsk::Utils.escape_shell_value("${PWD}")
+    assert_equal "\"${cat /etc/hostname}\"", Mrsk::Utils.escape_shell_value("${cat /etc/hostname}")
+    assert_equal "\"\\${PWD]\"", Mrsk::Utils.escape_shell_value("${PWD]")
+    assert_equal "\"\\$(PWD)\"", Mrsk::Utils.escape_shell_value("$(PWD)")
+    assert_equal "\"\\$PWD\"", Mrsk::Utils.escape_shell_value("$PWD")
+
+    assert_equal "\"^(https?://)www.example.com/(.*)\\$\"",
+      Mrsk::Utils.escape_shell_value("^(https?://)www.example.com/(.*)$")
+    assert_equal "\"https://example.com/\\$2\"",
+      Mrsk::Utils.escape_shell_value("https://example.com/$2")
  end
 end