Add more integration tests

Add tests for main, app, accessory, traefik and lock commands.
Other commands are generally covered by the main tests.

Also adds some changes to speed up the integration specs:
- Use a persistent volume for the registry so we can push images to to
reuse between runs (also gets around docker hub rate limits)
- Use persistent volume for mrsk gem install, to avoid re-installing
between tests
- Shorter stop wait time
- Shorter connection timeouts on the load balancer

Takes just over 2 minutes to run all tests locally on an M1 Mac
after docker caches are primed.

lib/mrsk/cli/main.rb

@@ -224,6 +224,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "healthcheck", "Healthcheck application"
   subcommand "healthcheck", Mrsk::Cli::Healthcheck
 
+  desc "lock", "Manage the deploy lock"
+  subcommand "lock", Mrsk::Cli::Lock
+
   desc "prune", "Prune old application images and containers"
   subcommand "prune", Mrsk::Cli::Prune
 
@@ -236,9 +239,6 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "traefik", "Manage Traefik load balancer"
   subcommand "traefik", Mrsk::Cli::Traefik
 
-  desc "lock", "Manage the deploy lock"
-  subcommand "lock", Mrsk::Cli::Lock
-
   private
     def container_available?(version)
       begin

test/cli/cli_test_case.rb

@@ -1,8 +1,6 @@
 require "test_helper"
 
 class CliTestCase < ActiveSupport::TestCase
-  include ActiveSupport::Testing::Stream
-
   setup do
     ENV["VERSION"]             = "999"
     ENV["RAILS_MASTER_KEY"]    = "123"

test/integration/accessory_test.rb

@@ -0,0 +1,36 @@
+require_relative "integration_test"
+
+class AccessoryTest < IntegrationTest
+  test "boot, stop, start, restart, logs, remove" do
+    mrsk :accessory, :boot, :busybox
+    assert_accessory_running :busybox
+
+    mrsk :accessory, :stop, :busybox
+    assert_accessory_not_running :busybox
+
+    mrsk :accessory, :start, :busybox
+    assert_accessory_running :busybox
+
+    mrsk :accessory, :restart, :busybox
+    assert_accessory_running :busybox
+
+    logs = mrsk :accessory, :logs, :busybox, capture: true
+    assert_match /Starting busybox.../, logs
+
+    mrsk :accessory, :remove, :busybox, "-y"
+    assert_accessory_not_running :busybox
+  end
+
+  private
+    def assert_accessory_running(name)
+      assert_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
+    end
+
+    def assert_accessory_not_running(name)
+      refute_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
+    end
+
+    def accessory_details(name)
+      mrsk :accessory, :details, name, capture: true
+    end
+end

test/integration/app_test.rb

@@ -0,0 +1,55 @@
+require_relative "integration_test"
+
+class AppTest < IntegrationTest
+  test "stop, start, boot, logs, images, containers, exec, remove" do
+    mrsk :deploy
+
+    assert_app_is_up
+
+    mrsk :app, :stop
+
+    # traefik is up and returns 404s when it can't match a route
+    assert_app_not_found
+
+    mrsk :app, :start
+
+    # mrsk app start does not wait
+    wait_for_app_to_be_up
+
+    mrsk :app, :boot
+
+    wait_for_app_to_be_up
+
+    logs = mrsk :app, :logs, capture: true
+    assert_match /App Host: vm1/, logs
+    assert_match /App Host: vm2/, logs
+    assert_match /GET \/ HTTP\/1.1/, logs
+
+    images = mrsk :app, :images, capture: true
+    assert_match /App Host: vm1/, images
+    assert_match /App Host: vm2/, images
+    assert_match /registry:4443\/app\s+#{latest_app_version}/, images
+    assert_match /registry:4443\/app\s+latest/, images
+
+    containers = mrsk :app, :containers, capture: true
+    assert_match /App Host: vm1/, containers
+    assert_match /App Host: vm2/, containers
+    assert_match /registry:4443\/app:#{latest_app_version}/, containers
+    assert_match /registry:4443\/app:latest/, containers
+
+    exec_output = mrsk :app, :exec, :ps, capture: true
+    assert_match /App Host: vm1/, exec_output
+    assert_match /App Host: vm2/, exec_output
+    assert_match /1 root      0:\d\d ps/, exec_output
+
+    exec_output = mrsk :app, :exec, "--reuse", :ps, capture: true
+    assert_match /App Host: vm1/, exec_output
+    assert_match /App Host: vm2/, exec_output
+    assert_match /1 root      0:\d\d nginx/, exec_output
+
+    mrsk :app, :remove
+
+    # traefik is up and returns 404s when it can't match a route
+    assert_app_not_found
+  end
+end

test/integration/docker-compose.yml

@@ -3,6 +3,8 @@ name: "mrsk-test"
 
 volumes:
   shared:
+  registry:
+  deployer_bundle:
 
 services:
   shared:
@@ -18,6 +20,8 @@ services:
     volumes:
       - ../..:/mrsk
       - shared:/shared
+      - registry:/registry
+      - deployer_bundle:/usr/local/bundle/
 
   registry:
     build:
@@ -28,6 +32,7 @@ services:
       - REGISTRY_HTTP_TLS_KEY=/certs/domain.key
     volumes:
       - shared:/shared
+      - registry:/var/lib/registry/
 
   vm1:
     privileged: true

test/integration/docker/deployer/app/.env.erb

@@ -0,0 +1 @@
+SECRET_TOKEN=1234

test/integration/docker/deployer/app/Dockerfile

@@ -1,4 +1,4 @@
-FROM nginx:1-alpine-slim
+FROM registry:4443/nginx:1-alpine-slim
 
 COPY default.conf /etc/nginx/conf.d/default.conf
 

test/integration/docker/deployer/app/config/deploy.yml

@@ -17,3 +17,11 @@ traefik:
   args:
     accesslog: true
     accesslog.format: json
+  image: registry:4443/traefik:v2.9
+accessories:
+  busybox:
+    image: registry:4443/busybox:1.36.0
+    cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done'
+    roles:
+      - web
+stop_wait_time: 1

test/integration/docker/deployer/boot.sh

@@ -1,9 +1,5 @@
 #!/bin/bash
 
-cd /mrsk && gem build mrsk.gemspec -o /tmp/mrsk.gem && gem install /tmp/mrsk.gem
-
 dockerd &
 
-trap "pkill -f sleep" term
+exec sleep infinity
-
-sleep infinity & wait

test/integration/docker/deployer/setup.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+install_mrsk() {
+  cd /mrsk && gem build mrsk.gemspec -o /tmp/mrsk.gem && gem install /tmp/mrsk.gem
+}
+
+# Push the images to a persistent volume on the registry container
+# This is to work around docker hub rate limits
+push_image_to_registry_4443() {
+  # Check if the image is in the registry without having to pull it
+  if ! stat /registry/docker/registry/v2/repositories/$1/_manifests/tags/$2/current/link > /dev/null; then
+    hub_tag=$1:$2
+    registry_4443_tag=registry:4443/$1:$2
+    docker pull $hub_tag
+    docker tag $hub_tag $registry_4443_tag
+    docker push $registry_4443_tag
+  fi
+}
+
+install_mrsk
+push_image_to_registry_4443 nginx 1-alpine-slim
+push_image_to_registry_4443 traefik v2.9
+push_image_to_registry_4443 busybox 1.36.0

test/integration/docker/load_balancer/default.conf

@@ -8,5 +8,9 @@ server {
 
   location / {
     proxy_pass http://loadbalancer;
+    proxy_connect_timeout       5;
+    proxy_send_timeout          5;
+    proxy_read_timeout          5;
+    send_timeout                5;
   }
 }

test/integration/docker/registry/boot.sh

@@ -2,6 +2,4 @@
 
 while [ ! -f /certs/domain.crt ]; do sleep 1; done
 
-trap "pkill -f registry" term
+exec /entrypoint.sh /etc/docker/registry/config.yml
-
-/entrypoint.sh /etc/docker/registry/config.yml & wait

test/integration/docker/shared/boot.sh

@@ -2,6 +2,4 @@
 
 cp -r * /shared
 
-trap "pkill -f sleep" term
+exec sleep infinity
-
-sleep infinity & wait

test/integration/docker/vm/boot.sh

@@ -6,6 +6,4 @@ service ssh restart
 
 dockerd &
 
-trap "pkill -f sleep" term
+exec sleep infinity
-
-sleep infinity & wait

test/integration/integration_test.rb

@@ -1,52 +1,19 @@
 require "net/http"
 require "test_helper"
 
-class DeployTest < ActiveSupport::TestCase
+class IntegrationTest < ActiveSupport::TestCase
-
   setup do
-    docker_compose "up --build --force-recreate -d"
+    docker_compose "up --build -d"
     wait_for_healthy
+    setup_deployer
   end
 
   teardown do
-    docker_compose "down -v"
+    docker_compose "down -t 1"
-  end
-
-  test "deploy" do
-    first_version = latest_app_version
-
-    assert_app_is_down
-
-    mrsk :deploy
-
-    assert_app_is_up version: first_version
-
-    second_version = update_app_rev
-
-    mrsk :redeploy
-
-    assert_app_is_up version: second_version
-
-    mrsk :rollback, first_version
-
-    assert_app_is_up version: first_version
-
-    details = mrsk :details, capture: true
-
-    assert_match /Traefik Host: vm1/, details
-    assert_match /Traefik Host: vm2/, details
-    assert_match /App Host: vm1/, details
-    assert_match /App Host: vm2/, details
-    assert_match /traefik:v2.9/, details
-    assert_match /registry:4443\/app:#{first_version}/, details
-
-    audit = mrsk :audit, capture: true
-
-    assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
   end
 
   private
-    def docker_compose(*commands, capture: false)
+    def docker_compose(*commands, capture: false, raise_on_error: true)
       command = "docker compose #{commands.join(" ")}"
       succeeded = false
       if capture
@@ -55,7 +22,7 @@ class DeployTest < ActiveSupport::TestCase
         succeeded = system("cd test/integration && #{command}")
       end
 
-      raise "Command `#{command}` failed with error code `#{$?}`" unless succeeded
+      raise "Command `#{command}` failed with error code `#{$?}`" if !succeeded && raise_on_error
       result
     end
 
@@ -68,24 +35,22 @@ class DeployTest < ActiveSupport::TestCase
     end
 
     def assert_app_is_down
-      assert_equal "502", app_response.code
+      response = app_response
+      debug_response_code(response, "502")
+      assert_equal "502", response.code
     end
 
     def assert_app_is_up(version: nil)
-      code = app_response.code
+      response = app_response
-      if code != "200"
+      debug_response_code(response, "200")
-        puts "Got response code #{code}, here are the traefik logs:"
+      assert_equal "200", response.code
-        mrsk :traefik, :logs
+      assert_app_version(version, response) if version
-        puts "And here are the load balancer logs"
-        docker_compose :logs, :load_balancer
-        puts "Tried to get the response code again and got #{app_response.code}"
-      end
-      assert_equal "200", code
-      assert_app_version(version) if version
     end
 
     def assert_app_not_found
-      assert_equal "404", app_response.code
+      response = app_response
+      debug_response_code(response, "404")
+      assert_equal "404", response.code
     end
 
     def wait_for_app_to_be_up(timeout: 10, up_count: 3)
@@ -101,7 +66,7 @@ class DeployTest < ActiveSupport::TestCase
     end
 
     def app_response
-      Net::HTTP.get_response(URI.parse("http://localhost:12345"))
+      Net::HTTP.get_response(URI.parse("http://localhost:12345/version"))
     end
 
     def update_app_rev
@@ -113,10 +78,8 @@ class DeployTest < ActiveSupport::TestCase
       deployer_exec("git rev-parse HEAD", capture: true)
     end
 
-    def assert_app_version(version)
+    def assert_app_version(version, response)
-      actual_version = Net::HTTP.get_response(URI.parse("http://localhost:12345/version")).body.strip
+      assert_equal version, response.body.strip
-
-      assert_equal version, actual_version
     end
 
     def wait_for_healthy(timeout: 20)
@@ -129,4 +92,20 @@ class DeployTest < ActiveSupport::TestCase
         sleep 0.1
       end
     end
+
+    def setup_deployer
+      deployer_exec("./setup.sh") unless $DEPLOYER_SETUP
+      $DEPLOYER_SETUP = true
+    end
+
+    def debug_response_code(app_response, expected_code)
+      code = app_response.code
+      if code != expected_code
+        puts "Got response code #{code}, here are the traefik logs:"
+        mrsk :traefik, :logs
+        puts "And here are the load balancer logs"
+        docker_compose :logs, :load_balancer
+        puts "Tried to get the response code again and got #{app_response.code}"
+      end
+    end
 end

test/integration/lock_test.rb

@@ -0,0 +1,18 @@
+require_relative "integration_test"
+
+class LockTest < IntegrationTest
+  test "acquire, release, status" do
+    mrsk :lock, :acquire, "-m 'Integration Tests'"
+
+    status = mrsk :lock, :status, capture: true
+    assert_match /Locked by: Deployer at .*\nVersion: #{latest_app_version}\nMessage: Integration Tests/m, status
+
+    error = mrsk :deploy, capture: true, raise_on_error: false
+    assert_match /Deploy lock found/m, error
+
+    mrsk :lock, :release
+
+    status = mrsk :lock, :status, capture: true
+    assert_match /There is no deploy lock/m, status
+  end
+end

test/integration/main_test.rb

@@ -0,0 +1,61 @@
+require_relative "integration_test"
+
+class MainTest < IntegrationTest
+  test "deploy, redeploy, rollback, details and audit" do
+    first_version = latest_app_version
+
+    assert_app_is_down
+
+    mrsk :deploy
+
+    assert_app_is_up version: first_version
+
+    second_version = update_app_rev
+
+    mrsk :redeploy
+
+    assert_app_is_up version: second_version
+
+    mrsk :rollback, first_version
+
+    assert_app_is_up version: first_version
+
+    details = mrsk :details, capture: true
+
+    assert_match /Traefik Host: vm1/, details
+    assert_match /Traefik Host: vm2/, details
+    assert_match /App Host: vm1/, details
+    assert_match /App Host: vm2/, details
+    assert_match /traefik:v2.9/, details
+    assert_match /registry:4443\/app:#{first_version}/, details
+
+    audit = mrsk :audit, capture: true
+
+    assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
+  end
+
+  test "envify" do
+    mrsk :envify
+
+    assert_equal "SECRET_TOKEN=1234", deployer_exec("cat .env", capture: true)
+  end
+
+  test "config" do
+    config = YAML.load(mrsk(:config, capture: true))
+    version = latest_app_version
+
+    assert_equal [ "web" ], config[:roles]
+    assert_equal [ "vm1", "vm2" ], config[:hosts]
+    assert_equal "vm1", config[:primary_host]
+    assert_equal version, config[:version]
+    assert_equal "registry:4443/app", config[:repository]
+    assert_equal "registry:4443/app:#{version}", config[:absolute_image]
+    assert_equal "app-#{version}", config[:service_with_version]
+    assert_equal [], config[:env_args]
+    assert_equal [], config[:volume_args]
+    assert_equal({ user: "root", auth_methods: [ "publickey" ] }, config[:ssh_options])
+    assert_equal({ "multiarch" => false, "args" => { "COMMIT_SHA" => version } }, config[:builder])
+    assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging]
+    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "cmd" => "wget -qO- http://localhost > /dev/null" }, config[:healthcheck])
+  end
+end

test/integration/traefik_test.rb

@@ -0,0 +1,36 @@
+require_relative "integration_test"
+
+class TraefikTest < IntegrationTest
+  test "boot, stop, start, restart, logs, remove" do
+    mrsk :traefik, :boot
+    assert_traefik_running
+
+    mrsk :traefik, :stop
+    assert_traefik_not_running
+
+    mrsk :traefik, :start
+    assert_traefik_running
+
+    mrsk :traefik, :restart
+    assert_traefik_running
+
+    logs = mrsk :traefik, :logs, capture: true
+    assert_match /Traefik version [\d.]+ built on/, logs
+
+    mrsk :traefik, :remove
+    assert_traefik_not_running
+  end
+
+  private
+    def assert_traefik_running
+      assert_match /traefik:v2.9   "\/entrypoint.sh/, traefik_details
+    end
+
+    def assert_traefik_not_running
+      refute_match /traefik:v2.9   "\/entrypoint.sh/, traefik_details
+    end
+
+    def traefik_details
+      mrsk :traefik, :details, capture: true
+    end
+end