Bump version for 0.7.0

Add healthcheck before deploy

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    mrsk (0.6.4)
+    mrsk (0.7.0)
       activesupport (>= 7.0)
       dotenv (~> 2.8)
       sshkit (~> 1.21)

README.md

@@ -22,8 +22,6 @@ env:
 
 Then edit your `.env` file to add your registry password as `MRSK_REGISTRY_PASSWORD` (and your `RAILS_MASTER_KEY` for production with a Rails app). 
 
-Finally, you have to ensure your application can answer `200 OK` to a `GET /up` request. That's how the zero-downtime deploy process knows that your new version is ready to serve traffic.
-
 Now you're ready to deploy to the servers:
 
 ```
@@ -39,9 +37,10 @@ This will:
 5. Push the image to the registry.
 6. Pull the image from the registry on the servers.
 7. Ensure Traefik is running and accepting traffic on port 80.
-8. Stop any containers running a previous versions of the app.
+8. Ensure your app responds with `200 OK` to `GET /up`.
-9. Start a new container with the version of the app that matches the current git version hash.
+9. Stop any containers running a previous versions of the app.
-10. Prune unused images and stopped containers to ensure servers don't fill up.
+10. Start a new container with the version of the app that matches the current git version hash.
+11. Prune unused images and stopped containers to ensure servers don't fill up.
 
 Voila! All the servers are now serving the app on port 80. If you're just running a single server, you're ready to go. If you're running multiple servers, you need to put a load balancer in front of them.
 
@@ -185,10 +184,10 @@ You can specialize the default Traefik rules by setting labels on the containers
 
 ```
 labels:
-  traefik.http.routers.hey.rule: '''Host(`app.hey.com`)'''
+  traefik.http.routers.hey.rule: Host(\`app.hey.com\`)
 ```
 
-Note: The extra quotes are needed to ensure the rule is passed in correctly!
+Note: The escaped backticks are needed to ensure the rule is passed in correctly and not treated as command substitution by Bash!
 
 This allows you to run multiple applications on the same server sharing the same Traefik instance and port.
 See https://doc.traefik.io/traefik/routing/routers/#rule for a full list of available routing rules.
@@ -370,6 +369,18 @@ That'll post a line like follows to a preconfigured chatbot in Basecamp:
 [My App] [2023-02-18 11:29:52] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de
 ```
 
+### Using custom healthcheck path or port
+
+MRSK defaults to checking the health of your application again `/up` on port 3000. You can tailor both with the `healthcheck` setting:
+
+```yaml
+healthcheck:
+  path: /healthz
+  port: 4000
+```
+
+This will ensure your application is configured with a traefik label for the healthcheck against `/healthz` and that the pre-deploy healthcheck that MRSK performs is done against the same path on port 4000.
+
 ## Commands
 
 ### Running commands on servers

lib/mrsk/cli/healthcheck.rb

@@ -0,0 +1,29 @@
+class Mrsk::Cli::Healthcheck < Mrsk::Cli::Base
+  desc "perform", "Health check the current version of the app"
+  def perform
+    on(MRSK.primary_host) do
+      begin
+        execute *MRSK.healthcheck.run
+
+        target = "Health check against #{MRSK.config.healthcheck["path"]}"
+
+        if capture_with_info(*MRSK.healthcheck.curl) == "200"
+          info "#{target} succeeded with 200 OK!"
+        else
+          # Catches 1xx, 2xx, 3xx
+          raise SSHKit::Command::Failed, "#{target} failed to return 200 OK!"
+        end
+      rescue SSHKit::Command::Failed => e
+        if e.message =~ /curl/
+          # Catches 4xx, 5xx
+          raise SSHKit::Command::Failed, "#{target} failed to return 200 OK!"
+        else
+          raise
+        end
+      ensure
+        execute *MRSK.healthcheck.stop, raise_on_non_zero_exit: false
+        execute *MRSK.healthcheck.remove, raise_on_non_zero_exit: false
+      end
+    end
+  end
+end

lib/mrsk/cli/main.rb

@@ -23,13 +23,16 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       say "Ensure Traefik is running...", :magenta
       invoke "mrsk:cli:traefik:boot"
 
+      say "Ensure app can pass healthcheck...", :magenta
+      invoke "mrsk:cli:healthcheck:perform"
+
       invoke "mrsk:cli:app:boot"
 
       say "Prune old containers and images...", :magenta
       invoke "mrsk:cli:prune:all"
     end
 
-    audit_broadcast "Deployed in #{runtime.to_i} seconds"
+    audit_broadcast "Deployed app in #{runtime.to_i} seconds"
   end
 
   desc "redeploy", "Deploy new version of the app to servers (without bootstrapping servers, starting Traefik, pruning, and registry login)"
@@ -38,25 +41,31 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       say "Build and push app image...", :magenta
       invoke "mrsk:cli:build:deliver"
 
+      say "Ensure app can pass healthcheck...", :magenta
+      invoke "mrsk:cli:healthcheck:perform"
+
       invoke "mrsk:cli:app:boot"
     end
 
-    audit_broadcast "Redeployed in #{runtime.to_i} seconds"
+    audit_broadcast "Redeployed app in #{runtime.to_i} seconds"
   end
 
   desc "rollback [VERSION]", "Rollback the app to VERSION"
   def rollback(version)
     MRSK.version = version
 
-    cli = self
+    if container_name_available?(MRSK.config.service_with_version)
+      say "Stop current version, then start version #{version}...", :magenta
 
-    cli.say "Stop current version, then start version #{version}...", :magenta
+      on(MRSK.hosts) do |host|
-    on(MRSK.hosts) do
         execute *MRSK.app.stop, raise_on_non_zero_exit: false
         execute *MRSK.app.start
       end
 
-    audit_broadcast "Rolled back to version #{version}"
+      audit_broadcast "Rolled back app to version #{version}"
+    else
+      say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
+    end
   end
 
   desc "details", "Display details about Traefik and app containers"
@@ -144,6 +153,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "build", "Build the application image"
   subcommand "build", Mrsk::Cli::Build
 
+  desc "healthcheck", "Healthcheck the application"
+  subcommand "healthcheck", Mrsk::Cli::Healthcheck
+
   desc "prune", "Prune old application images and containers"
   subcommand "prune", Mrsk::Cli::Prune
 
@@ -155,4 +167,11 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
 
   desc "traefik", "Manage the Traefik load balancer"
   subcommand "traefik", Mrsk::Cli::Traefik
+
+  private
+    def container_name_available?(container_name, host: MRSK.primary_host)
+      container_names = nil
+      on(host) { container_names = capture_with_info(*MRSK.app.list_container_names).split("\n") }
+      Array(container_names).include?(container_name)
+    end
 end

lib/mrsk/commander.rb

@@ -73,6 +73,10 @@ class Mrsk::Commander
     @auditor ||= Mrsk::Commands::Auditor.new(config)
   end
 
+  def healthcheck
+    @healthcheck ||= Mrsk::Commands::Healthcheck.new(config)
+  end
+
 
   def with_verbosity(level)
     old_level = self.verbosity

lib/mrsk/commands/app.rb

@@ -75,10 +75,6 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
     docker :ps, "-q", *service_filter
   end
 
-  def container_id_for(container_name:)
-    docker :container, :ls, "-a", "-f", "name=#{container_name}", "-q"
-  end
-
   def current_running_version
     # FIXME: Find more graceful way to extract the version from "app-version" than using sed and tail!
     pipe \
@@ -93,11 +89,21 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
       "head -n 1"
   end
 
+  def all_versions_from_available_containers
+    pipe \
+      docker(:image, :ls, "--format", '"{{.Tag}}"', config.repository),
+      "head -n 1"
+  end
+
 
   def list_containers
     docker :container, :ls, "-a", *service_filter
   end
 
+  def list_container_names
+    [ *list_containers, "--format", "'{{ .Names }}'" ]
+  end
+
   def remove_container(version:)
     pipe \
       container_id_for(container_name: service_with_version(version)),

lib/mrsk/commands/base.rb

@@ -17,6 +17,10 @@ module Mrsk::Commands
       end
     end
 
+    def container_id_for(container_name:)
+      docker :container, :ls, "-a", "-f", "name=#{container_name}", "-q"
+    end
+
     private
       def combine(*commands, by: "&&")
         commands

lib/mrsk/commands/healthcheck.rb

@@ -0,0 +1,46 @@
+class Mrsk::Commands::Healthcheck < Mrsk::Commands::Base
+  EXPOSED_PORT = 3999
+
+  def run
+    web = config.role(:web)
+
+    docker :run,
+      "-d",
+      "--name", container_name_with_version,
+      "-p", "#{EXPOSED_PORT}:#{config.healthcheck["port"]}",
+      "--label", "service=#{container_name}",
+      *web.env_args,
+      *config.volume_args,
+      config.absolute_image,
+      web.cmd
+  end
+
+  def curl
+    [ :curl, "--silent", "--output", "/dev/null", "--write-out", "'%{http_code}'", health_url ]
+  end
+
+  def stop
+    pipe \
+      container_id_for(container_name: container_name),
+      xargs(docker(:stop))
+  end
+
+  def remove
+    pipe \
+      container_id_for(container_name: container_name),
+      xargs(docker(:container, :rm))
+  end
+
+  private
+    def container_name
+      "healthcheck-#{config.service}"
+    end
+
+    def container_name_with_version
+      "healthcheck-#{config.service_with_version}"
+    end
+
+    def health_url
+      "http://localhost:#{EXPOSED_PORT}#{config.healthcheck["path"]}"
+    end
+end

lib/mrsk/configuration.rb

@@ -107,6 +107,7 @@ class Mrsk::Configuration
     end
   end
 
+
   def ssh_user
     if raw_config.ssh.present?
       raw_config.ssh["user"] || "root"
@@ -126,10 +127,15 @@ class Mrsk::Configuration
     { user: ssh_user, proxy: ssh_proxy, auth_methods: [ "publickey" ] }.compact
   end
 
+
   def audit_broadcast_cmd
     raw_config.audit_broadcast_cmd
   end
 
+  def healthcheck
+    { "path" => "/up", "port" => 3000 }.merge(raw_config.healthcheck || {})
+  end
+
 
   def valid?
     ensure_required_keys_present && ensure_env_available
@@ -149,7 +155,8 @@ class Mrsk::Configuration
       volume_args: volume_args,
       ssh_options: ssh_options,
       builder: raw_config.builder,
-      accessories: raw_config.accessories
+      accessories: raw_config.accessories,
+      healthcheck: healthcheck
     }.compact
   end
 

lib/mrsk/configuration/role.rb

@@ -59,7 +59,7 @@ class Mrsk::Configuration::Role
       if running_traefik?
         {
           "traefik.http.routers.#{config.service}.rule" => "'PathPrefix(`/`)'",
-          "traefik.http.services.#{config.service}.loadbalancer.healthcheck.path" => "/up",
+          "traefik.http.services.#{config.service}.loadbalancer.healthcheck.path" => config.healthcheck["path"],
           "traefik.http.services.#{config.service}.loadbalancer.healthcheck.interval" => "1s",
           "traefik.http.middlewares.#{config.service}.retry.attempts" => "3",
           "traefik.http.middlewares.#{config.service}.retry.initialinterval" => "500ms"

lib/mrsk/version.rb

@@ -1,3 +1,3 @@
 module Mrsk
-  VERSION = "0.6.4"
+  VERSION = "0.7.0"
 end

test/cli/main_test.rb

@@ -5,4 +5,29 @@ class CliMainTest < CliTestCase
     version = stdouted { Mrsk::Cli::Main.new.version }
     assert_equal Mrsk::VERSION, version
   end
+
+  test "rollback bad version" do
+    run_command("details") # Preheat MRSK const
+
+    run_command("rollback", "nonsense").tap do |output|
+      assert_match /docker container ls -a --filter label=service=app --format '{{ .Names }}'/, output
+      assert_match /The app version 'nonsense' is not available as a container/, output
+    end
+  end
+
+  test "rollback good version" do
+    Mrsk::Cli::Main.any_instance.stubs(:container_name_available?).returns(true)
+
+    run_command("rollback", "123").tap do |output|
+      assert_match /Stop current version, then start version 123/, output
+      assert_match /docker ps -q --filter label=service=app | xargs docker stop/, output
+      assert_match /docker start app-123/, output
+    end
+  end
+
+
+  private
+    def run_command(*command)
+      stdouted { Mrsk::Cli::Main.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+    end
 end

test/commands/app_test.rb

@@ -26,6 +26,14 @@ class CommandsAppTest < ActiveSupport::TestCase
       @app.run.join(" ")
   end
 
+  test "run with custom healthcheck path" do
+    @config[:healthcheck] = { "path" => "/healthz" }
+
+    assert_equal \
+      "docker run -d --restart unless-stopped --log-opt max-size=10m --name app-999 -e RAILS_MASTER_KEY=456 --label service=app --label role=web --label traefik.http.routers.app.rule='PathPrefix(`/`)' --label traefik.http.services.app.loadbalancer.healthcheck.path=/healthz --label traefik.http.services.app.loadbalancer.healthcheck.interval=1s --label traefik.http.middlewares.app.retry.attempts=3 --label traefik.http.middlewares.app.retry.initialinterval=500ms dhh/app:999",
+      @app.run.join(" ")
+  end
+
   test "start" do
     assert_equal \
       "docker start app-999",

test/commands/healthcheck_test.rb

@@ -0,0 +1,55 @@
+require "test_helper"
+
+class CommandsHealthcheckTest < ActiveSupport::TestCase
+  setup do
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
+    }
+  end
+
+  test "run" do
+    assert_equal \
+      "docker run -d --name healthcheck-app-123 -p 3999:3000 --label service=healthcheck-app dhh/app:123",
+      new_command.run.join(" ")
+  end
+
+  test "run with custom port" do
+    @config[:healthcheck] = { "port" => 3001 }
+
+    assert_equal \
+      "docker run -d --name healthcheck-app-123 -p 3999:3001 --label service=healthcheck-app dhh/app:123",
+      new_command.run.join(" ")
+  end
+
+  test "curl" do
+    assert_equal \
+      "curl --silent --output /dev/null --write-out '%{http_code}' http://localhost:3999/up",
+      new_command.curl.join(" ")
+  end
+
+  test "curl with custom path" do
+    @config[:healthcheck] = { "path" => "/healthz" }
+
+    assert_equal \
+      "curl --silent --output /dev/null --write-out '%{http_code}' http://localhost:3999/healthz",
+      new_command.curl.join(" ")
+  end
+
+  test "stop" do
+    assert_equal \
+      "docker container ls -a -f name=healthcheck-app -q | xargs docker stop",
+      new_command.stop.join(" ")
+  end
+
+  test "remove" do
+    assert_equal \
+      "docker container ls -a -f name=healthcheck-app -q | xargs docker container rm",
+      new_command.remove.join(" ")
+  end
+
+  private
+    def new_command
+      Mrsk::Commands::Healthcheck.new(Mrsk::Configuration.new(@config, version: "123"))
+    end
+end