From 7cd25fd163e096122de8650fc59cf20cf21a5670 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Mon, 24 Apr 2023 14:34:46 +0100
Subject: [PATCH] Add more integration tests

Add tests for main, app, accessory, traefik and lock commands.
Other commands are generally covered by the main tests.

Also adds some changes to speed up the integration specs:
- Use a persistent volume for the registry so we can push images to to
reuse between runs (also gets around docker hub rate limits)
- Use persistent volume for mrsk gem install, to avoid re-installing
between tests
- Shorter stop wait time
- Shorter connection timeouts on the load balancer

Takes just over 2 minutes to run all tests locally on an M1 Mac
after docker caches are primed.
---
 lib/mrsk/cli/main.rb                          |  6 +-
 test/cli/cli_test_case.rb                     |  2 -
 test/integration/accessory_test.rb            | 36 ++++++++
 test/integration/app_test.rb                  | 55 +++++++++++
 test/integration/docker-compose.yml           |  5 +
 test/integration/docker/deployer/app/.env.erb |  1 +
 .../docker/deployer/app/Dockerfile            |  2 +-
 .../docker/deployer/app/config/deploy.yml     |  8 ++
 test/integration/docker/deployer/boot.sh      |  6 +-
 test/integration/docker/deployer/setup.sh     | 23 +++++
 .../docker/load_balancer/default.conf         |  4 +
 test/integration/docker/registry/boot.sh      |  4 +-
 test/integration/docker/shared/boot.sh        |  4 +-
 test/integration/docker/vm/boot.sh            |  4 +-
 .../{deploy_test.rb => integration_test.rb}   | 91 +++++++------------
 test/integration/lock_test.rb                 | 18 ++++
 test/integration/main_test.rb                 | 61 +++++++++++++
 test/integration/traefik_test.rb              | 36 ++++++++
 18 files changed, 290 insertions(+), 76 deletions(-)
 create mode 100644 test/integration/accessory_test.rb
 create mode 100644 test/integration/app_test.rb
 create mode 100644 test/integration/docker/deployer/app/.env.erb
 create mode 100755 test/integration/docker/deployer/setup.sh
 rename test/integration/{deploy_test.rb => integration_test.rb} (59%)
 create mode 100644 test/integration/lock_test.rb
 create mode 100644 test/integration/main_test.rb
 create mode 100644 test/integration/traefik_test.rb

diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb
index 18ce3600..520a324b 100644
--- a/lib/mrsk/cli/main.rb
+++ b/lib/mrsk/cli/main.rb
@@ -224,6 +224,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "healthcheck", "Healthcheck application"
   subcommand "healthcheck", Mrsk::Cli::Healthcheck
 
+  desc "lock", "Manage the deploy lock"
+  subcommand "lock", Mrsk::Cli::Lock
+
   desc "prune", "Prune old application images and containers"
   subcommand "prune", Mrsk::Cli::Prune
 
@@ -236,9 +239,6 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "traefik", "Manage Traefik load balancer"
   subcommand "traefik", Mrsk::Cli::Traefik
 
-  desc "lock", "Manage the deploy lock"
-  subcommand "lock", Mrsk::Cli::Lock
-
   private
     def container_available?(version)
       begin
diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb
index abd805ec..f582cc75 100644
--- a/test/cli/cli_test_case.rb
+++ b/test/cli/cli_test_case.rb
@@ -1,8 +1,6 @@
 require "test_helper"
 
 class CliTestCase < ActiveSupport::TestCase
-  include ActiveSupport::Testing::Stream
-
   setup do
     ENV["VERSION"]             = "999"
     ENV["RAILS_MASTER_KEY"]    = "123"
diff --git a/test/integration/accessory_test.rb b/test/integration/accessory_test.rb
new file mode 100644
index 00000000..151861a0
--- /dev/null
+++ b/test/integration/accessory_test.rb
@@ -0,0 +1,36 @@
+require_relative "integration_test"
+
+class AccessoryTest < IntegrationTest
+  test "boot, stop, start, restart, logs, remove" do
+    mrsk :accessory, :boot, :busybox
+    assert_accessory_running :busybox
+
+    mrsk :accessory, :stop, :busybox
+    assert_accessory_not_running :busybox
+
+    mrsk :accessory, :start, :busybox
+    assert_accessory_running :busybox
+
+    mrsk :accessory, :restart, :busybox
+    assert_accessory_running :busybox
+
+    logs = mrsk :accessory, :logs, :busybox, capture: true
+    assert_match /Starting busybox.../, logs
+
+    mrsk :accessory, :remove, :busybox, "-y"
+    assert_accessory_not_running :busybox
+  end
+
+  private
+    def assert_accessory_running(name)
+      assert_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
+    end
+
+    def assert_accessory_not_running(name)
+      refute_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
+    end
+
+    def accessory_details(name)
+      mrsk :accessory, :details, name, capture: true
+    end
+end
diff --git a/test/integration/app_test.rb b/test/integration/app_test.rb
new file mode 100644
index 00000000..13ba65dc
--- /dev/null
+++ b/test/integration/app_test.rb
@@ -0,0 +1,55 @@
+require_relative "integration_test"
+
+class AppTest < IntegrationTest
+  test "stop, start, boot, logs, images, containers, exec, remove" do
+    mrsk :deploy
+
+    assert_app_is_up
+
+    mrsk :app, :stop
+
+    # traefik is up and returns 404s when it can't match a route
+    assert_app_not_found
+
+    mrsk :app, :start
+
+    # mrsk app start does not wait
+    wait_for_app_to_be_up
+
+    mrsk :app, :boot
+
+    wait_for_app_to_be_up
+
+    logs = mrsk :app, :logs, capture: true
+    assert_match /App Host: vm1/, logs
+    assert_match /App Host: vm2/, logs
+    assert_match /GET \/ HTTP\/1.1/, logs
+
+    images = mrsk :app, :images, capture: true
+    assert_match /App Host: vm1/, images
+    assert_match /App Host: vm2/, images
+    assert_match /registry:4443\/app\s+#{latest_app_version}/, images
+    assert_match /registry:4443\/app\s+latest/, images
+
+    containers = mrsk :app, :containers, capture: true
+    assert_match /App Host: vm1/, containers
+    assert_match /App Host: vm2/, containers
+    assert_match /registry:4443\/app:#{latest_app_version}/, containers
+    assert_match /registry:4443\/app:latest/, containers
+
+    exec_output = mrsk :app, :exec, :ps, capture: true
+    assert_match /App Host: vm1/, exec_output
+    assert_match /App Host: vm2/, exec_output
+    assert_match /1 root      0:\d\d ps/, exec_output
+
+    exec_output = mrsk :app, :exec, "--reuse", :ps, capture: true
+    assert_match /App Host: vm1/, exec_output
+    assert_match /App Host: vm2/, exec_output
+    assert_match /1 root      0:\d\d nginx/, exec_output
+
+    mrsk :app, :remove
+
+    # traefik is up and returns 404s when it can't match a route
+    assert_app_not_found
+  end
+end
diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml
index c9fdf306..a3b2db66 100644
--- a/test/integration/docker-compose.yml
+++ b/test/integration/docker-compose.yml
@@ -3,6 +3,8 @@ name: "mrsk-test"
 
 volumes:
   shared:
+  registry:
+  deployer_bundle:
 
 services:
   shared:
@@ -18,6 +20,8 @@ services:
     volumes:
       - ../..:/mrsk
       - shared:/shared
+      - registry:/registry
+      - deployer_bundle:/usr/local/bundle/
 
   registry:
     build:
@@ -28,6 +32,7 @@ services:
       - REGISTRY_HTTP_TLS_KEY=/certs/domain.key
     volumes:
       - shared:/shared
+      - registry:/var/lib/registry/
 
   vm1:
     privileged: true
diff --git a/test/integration/docker/deployer/app/.env.erb b/test/integration/docker/deployer/app/.env.erb
new file mode 100644
index 00000000..dcd2fcf5
--- /dev/null
+++ b/test/integration/docker/deployer/app/.env.erb
@@ -0,0 +1 @@
+SECRET_TOKEN=1234
diff --git a/test/integration/docker/deployer/app/Dockerfile b/test/integration/docker/deployer/app/Dockerfile
index f1b7b3d0..cbe60ae0 100644
--- a/test/integration/docker/deployer/app/Dockerfile
+++ b/test/integration/docker/deployer/app/Dockerfile
@@ -1,4 +1,4 @@
-FROM nginx:1-alpine-slim
+FROM registry:4443/nginx:1-alpine-slim
 
 COPY default.conf /etc/nginx/conf.d/default.conf
 
diff --git a/test/integration/docker/deployer/app/config/deploy.yml b/test/integration/docker/deployer/app/config/deploy.yml
index a06c17dc..bc2cb28c 100644
--- a/test/integration/docker/deployer/app/config/deploy.yml
+++ b/test/integration/docker/deployer/app/config/deploy.yml
@@ -17,3 +17,11 @@ traefik:
   args:
     accesslog: true
     accesslog.format: json
+  image: registry:4443/traefik:v2.9
+accessories:
+  busybox:
+    image: registry:4443/busybox:1.36.0
+    cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done'
+    roles:
+      - web
+stop_wait_time: 1
diff --git a/test/integration/docker/deployer/boot.sh b/test/integration/docker/deployer/boot.sh
index ac42c53a..b25f3334 100755
--- a/test/integration/docker/deployer/boot.sh
+++ b/test/integration/docker/deployer/boot.sh
@@ -1,9 +1,5 @@
 #!/bin/bash
 
-cd /mrsk && gem build mrsk.gemspec -o /tmp/mrsk.gem && gem install /tmp/mrsk.gem
-
 dockerd &
 
-trap "pkill -f sleep" term
-
-sleep infinity & wait
+exec sleep infinity
diff --git a/test/integration/docker/deployer/setup.sh b/test/integration/docker/deployer/setup.sh
new file mode 100755
index 00000000..3a0a8da7
--- /dev/null
+++ b/test/integration/docker/deployer/setup.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+install_mrsk() {
+  cd /mrsk && gem build mrsk.gemspec -o /tmp/mrsk.gem && gem install /tmp/mrsk.gem
+}
+
+# Push the images to a persistent volume on the registry container
+# This is to work around docker hub rate limits
+push_image_to_registry_4443() {
+  # Check if the image is in the registry without having to pull it
+  if ! stat /registry/docker/registry/v2/repositories/$1/_manifests/tags/$2/current/link > /dev/null; then
+    hub_tag=$1:$2
+    registry_4443_tag=registry:4443/$1:$2
+    docker pull $hub_tag
+    docker tag $hub_tag $registry_4443_tag
+    docker push $registry_4443_tag
+  fi
+}
+
+install_mrsk
+push_image_to_registry_4443 nginx 1-alpine-slim
+push_image_to_registry_4443 traefik v2.9
+push_image_to_registry_4443 busybox 1.36.0
diff --git a/test/integration/docker/load_balancer/default.conf b/test/integration/docker/load_balancer/default.conf
index 3b734e36..12d241a7 100644
--- a/test/integration/docker/load_balancer/default.conf
+++ b/test/integration/docker/load_balancer/default.conf
@@ -8,5 +8,9 @@ server {
 
   location / {
     proxy_pass http://loadbalancer;
+    proxy_connect_timeout       5;
+    proxy_send_timeout          5;
+    proxy_read_timeout          5;
+    send_timeout                5;
   }
 }
diff --git a/test/integration/docker/registry/boot.sh b/test/integration/docker/registry/boot.sh
index 411bc617..895838f5 100755
--- a/test/integration/docker/registry/boot.sh
+++ b/test/integration/docker/registry/boot.sh
@@ -2,6 +2,4 @@
 
 while [ ! -f /certs/domain.crt ]; do sleep 1; done
 
-trap "pkill -f registry" term
-
-/entrypoint.sh /etc/docker/registry/config.yml & wait
+exec /entrypoint.sh /etc/docker/registry/config.yml
diff --git a/test/integration/docker/shared/boot.sh b/test/integration/docker/shared/boot.sh
index 821c8c30..a9cba216 100755
--- a/test/integration/docker/shared/boot.sh
+++ b/test/integration/docker/shared/boot.sh
@@ -2,6 +2,4 @@
 
 cp -r * /shared
 
-trap "pkill -f sleep" term
-
-sleep infinity & wait
+exec sleep infinity
diff --git a/test/integration/docker/vm/boot.sh b/test/integration/docker/vm/boot.sh
index 5a26ab2e..81df5cdb 100755
--- a/test/integration/docker/vm/boot.sh
+++ b/test/integration/docker/vm/boot.sh
@@ -6,6 +6,4 @@ service ssh restart
 
 dockerd &
 
-trap "pkill -f sleep" term
-
-sleep infinity & wait
+exec sleep infinity
diff --git a/test/integration/deploy_test.rb b/test/integration/integration_test.rb
similarity index 59%
rename from test/integration/deploy_test.rb
rename to test/integration/integration_test.rb
index 8feb2beb..eaf2252b 100644
--- a/test/integration/deploy_test.rb
+++ b/test/integration/integration_test.rb
@@ -1,52 +1,19 @@
 require "net/http"
 require "test_helper"
 
-class DeployTest < ActiveSupport::TestCase
-
+class IntegrationTest < ActiveSupport::TestCase
   setup do
-    docker_compose "up --build --force-recreate -d"
+    docker_compose "up --build -d"
     wait_for_healthy
+    setup_deployer
   end
 
   teardown do
-    docker_compose "down -v"
-  end
-
-  test "deploy" do
-    first_version = latest_app_version
-
-    assert_app_is_down
-
-    mrsk :deploy
-
-    assert_app_is_up version: first_version
-
-    second_version = update_app_rev
-
-    mrsk :redeploy
-
-    assert_app_is_up version: second_version
-
-    mrsk :rollback, first_version
-
-    assert_app_is_up version: first_version
-
-    details = mrsk :details, capture: true
-
-    assert_match /Traefik Host: vm1/, details
-    assert_match /Traefik Host: vm2/, details
-    assert_match /App Host: vm1/, details
-    assert_match /App Host: vm2/, details
-    assert_match /traefik:v2.9/, details
-    assert_match /registry:4443\/app:#{first_version}/, details
-
-    audit = mrsk :audit, capture: true
-
-    assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
+    docker_compose "down -t 1"
   end
 
   private
-    def docker_compose(*commands, capture: false)
+    def docker_compose(*commands, capture: false, raise_on_error: true)
       command = "docker compose #{commands.join(" ")}"
       succeeded = false
       if capture
@@ -55,7 +22,7 @@ class DeployTest < ActiveSupport::TestCase
         succeeded = system("cd test/integration && #{command}")
       end
 
-      raise "Command `#{command}` failed with error code `#{$?}`" unless succeeded
+      raise "Command `#{command}` failed with error code `#{$?}`" if !succeeded && raise_on_error
       result
     end
 
@@ -68,24 +35,22 @@ class DeployTest < ActiveSupport::TestCase
     end
 
     def assert_app_is_down
-      assert_equal "502", app_response.code
+      response = app_response
+      debug_response_code(response, "502")
+      assert_equal "502", response.code
     end
 
     def assert_app_is_up(version: nil)
-      code = app_response.code
-      if code != "200"
-        puts "Got response code #{code}, here are the traefik logs:"
-        mrsk :traefik, :logs
-        puts "And here are the load balancer logs"
-        docker_compose :logs, :load_balancer
-        puts "Tried to get the response code again and got #{app_response.code}"
-      end
-      assert_equal "200", code
-      assert_app_version(version) if version
+      response = app_response
+      debug_response_code(response, "200")
+      assert_equal "200", response.code
+      assert_app_version(version, response) if version
     end
 
     def assert_app_not_found
-      assert_equal "404", app_response.code
+      response = app_response
+      debug_response_code(response, "404")
+      assert_equal "404", response.code
     end
 
     def wait_for_app_to_be_up(timeout: 10, up_count: 3)
@@ -101,7 +66,7 @@ class DeployTest < ActiveSupport::TestCase
     end
 
     def app_response
-      Net::HTTP.get_response(URI.parse("http://localhost:12345"))
+      Net::HTTP.get_response(URI.parse("http://localhost:12345/version"))
     end
 
     def update_app_rev
@@ -113,10 +78,8 @@ class DeployTest < ActiveSupport::TestCase
       deployer_exec("git rev-parse HEAD", capture: true)
     end
 
-    def assert_app_version(version)
-      actual_version = Net::HTTP.get_response(URI.parse("http://localhost:12345/version")).body.strip
-
-      assert_equal version, actual_version
+    def assert_app_version(version, response)
+      assert_equal version, response.body.strip
     end
 
     def wait_for_healthy(timeout: 20)
@@ -129,4 +92,20 @@ class DeployTest < ActiveSupport::TestCase
         sleep 0.1
       end
     end
+
+    def setup_deployer
+      deployer_exec("./setup.sh") unless $DEPLOYER_SETUP
+      $DEPLOYER_SETUP = true
+    end
+
+    def debug_response_code(app_response, expected_code)
+      code = app_response.code
+      if code != expected_code
+        puts "Got response code #{code}, here are the traefik logs:"
+        mrsk :traefik, :logs
+        puts "And here are the load balancer logs"
+        docker_compose :logs, :load_balancer
+        puts "Tried to get the response code again and got #{app_response.code}"
+      end
+    end
 end
diff --git a/test/integration/lock_test.rb b/test/integration/lock_test.rb
new file mode 100644
index 00000000..2740a249
--- /dev/null
+++ b/test/integration/lock_test.rb
@@ -0,0 +1,18 @@
+require_relative "integration_test"
+
+class LockTest < IntegrationTest
+  test "acquire, release, status" do
+    mrsk :lock, :acquire, "-m 'Integration Tests'"
+
+    status = mrsk :lock, :status, capture: true
+    assert_match /Locked by: Deployer at .*\nVersion: #{latest_app_version}\nMessage: Integration Tests/m, status
+
+    error = mrsk :deploy, capture: true, raise_on_error: false
+    assert_match /Deploy lock found/m, error
+
+    mrsk :lock, :release
+
+    status = mrsk :lock, :status, capture: true
+    assert_match /There is no deploy lock/m, status
+  end
+end
diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb
new file mode 100644
index 00000000..bd4c2dcc
--- /dev/null
+++ b/test/integration/main_test.rb
@@ -0,0 +1,61 @@
+require_relative "integration_test"
+
+class MainTest < IntegrationTest
+  test "deploy, redeploy, rollback, details and audit" do
+    first_version = latest_app_version
+
+    assert_app_is_down
+
+    mrsk :deploy
+
+    assert_app_is_up version: first_version
+
+    second_version = update_app_rev
+
+    mrsk :redeploy
+
+    assert_app_is_up version: second_version
+
+    mrsk :rollback, first_version
+
+    assert_app_is_up version: first_version
+
+    details = mrsk :details, capture: true
+
+    assert_match /Traefik Host: vm1/, details
+    assert_match /Traefik Host: vm2/, details
+    assert_match /App Host: vm1/, details
+    assert_match /App Host: vm2/, details
+    assert_match /traefik:v2.9/, details
+    assert_match /registry:4443\/app:#{first_version}/, details
+
+    audit = mrsk :audit, capture: true
+
+    assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
+  end
+
+  test "envify" do
+    mrsk :envify
+
+    assert_equal "SECRET_TOKEN=1234", deployer_exec("cat .env", capture: true)
+  end
+
+  test "config" do
+    config = YAML.load(mrsk(:config, capture: true))
+    version = latest_app_version
+
+    assert_equal [ "web" ], config[:roles]
+    assert_equal [ "vm1", "vm2" ], config[:hosts]
+    assert_equal "vm1", config[:primary_host]
+    assert_equal version, config[:version]
+    assert_equal "registry:4443/app", config[:repository]
+    assert_equal "registry:4443/app:#{version}", config[:absolute_image]
+    assert_equal "app-#{version}", config[:service_with_version]
+    assert_equal [], config[:env_args]
+    assert_equal [], config[:volume_args]
+    assert_equal({ user: "root", auth_methods: [ "publickey" ] }, config[:ssh_options])
+    assert_equal({ "multiarch" => false, "args" => { "COMMIT_SHA" => version } }, config[:builder])
+    assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging]
+    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "cmd" => "wget -qO- http://localhost > /dev/null" }, config[:healthcheck])
+  end
+end
diff --git a/test/integration/traefik_test.rb b/test/integration/traefik_test.rb
new file mode 100644
index 00000000..e74b8e16
--- /dev/null
+++ b/test/integration/traefik_test.rb
@@ -0,0 +1,36 @@
+require_relative "integration_test"
+
+class TraefikTest < IntegrationTest
+  test "boot, stop, start, restart, logs, remove" do
+    mrsk :traefik, :boot
+    assert_traefik_running
+
+    mrsk :traefik, :stop
+    assert_traefik_not_running
+
+    mrsk :traefik, :start
+    assert_traefik_running
+
+    mrsk :traefik, :restart
+    assert_traefik_running
+
+    logs = mrsk :traefik, :logs, capture: true
+    assert_match /Traefik version [\d.]+ built on/, logs
+
+    mrsk :traefik, :remove
+    assert_traefik_not_running
+  end
+
+  private
+    def assert_traefik_running
+      assert_match /traefik:v2.9   "\/entrypoint.sh/, traefik_details
+    end
+
+    def assert_traefik_not_running
+      refute_match /traefik:v2.9   "\/entrypoint.sh/, traefik_details
+    end
+
+    def traefik_details
+      mrsk :traefik, :details, capture: true
+    end
+end