Merge pull request #298 from basecamp/more-integration-tests

lib/mrsk/cli/main.rb

@@ -224,6 +224,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "healthcheck", "Healthcheck application"
   subcommand "healthcheck", Mrsk::Cli::Healthcheck
 
+  desc "lock", "Manage the deploy lock"
+  subcommand "lock", Mrsk::Cli::Lock
+
   desc "prune", "Prune old application images and containers"
   subcommand "prune", Mrsk::Cli::Prune
 
@@ -236,9 +239,6 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "traefik", "Manage Traefik load balancer"
   subcommand "traefik", Mrsk::Cli::Traefik
 
-  desc "lock", "Manage the deploy lock"
-  subcommand "lock", Mrsk::Cli::Lock
-
   private
     def container_available?(version)
       begin

test/cli/cli_test_case.rb

@@ -1,8 +1,6 @@
 require "test_helper"
 
 class CliTestCase < ActiveSupport::TestCase
-  include ActiveSupport::Testing::Stream
-
   setup do
     ENV["VERSION"]             = "999"
     ENV["RAILS_MASTER_KEY"]    = "123"

test/integration/accessory_test.rb

@@ -0,0 +1,36 @@
+require_relative "integration_test"
+
+class AccessoryTest < IntegrationTest
+  test "boot, stop, start, restart, logs, remove" do
+    mrsk :accessory, :boot, :busybox
+    assert_accessory_running :busybox
+
+    mrsk :accessory, :stop, :busybox
+    assert_accessory_not_running :busybox
+
+    mrsk :accessory, :start, :busybox
+    assert_accessory_running :busybox
+
+    mrsk :accessory, :restart, :busybox
+    assert_accessory_running :busybox
+
+    logs = mrsk :accessory, :logs, :busybox, capture: true
+    assert_match /Starting busybox.../, logs
+
+    mrsk :accessory, :remove, :busybox, "-y"
+    assert_accessory_not_running :busybox
+  end
+
+  private
+    def assert_accessory_running(name)
+      assert_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
+    end
+
+    def assert_accessory_not_running(name)
+      refute_match /registry:4443\/busybox:1.36.0   "sh -c 'echo \\"Start/, accessory_details(name)
+    end
+
+    def accessory_details(name)
+      mrsk :accessory, :details, name, capture: true
+    end
+end

test/integration/app_test.rb

@@ -0,0 +1,55 @@
+require_relative "integration_test"
+
+class AppTest < IntegrationTest
+  test "stop, start, boot, logs, images, containers, exec, remove" do
+    mrsk :deploy
+
+    assert_app_is_up
+
+    mrsk :app, :stop
+
+    # traefik is up and returns 404s when it can't match a route
+    assert_app_not_found
+
+    mrsk :app, :start
+
+    # mrsk app start does not wait
+    wait_for_app_to_be_up
+
+    mrsk :app, :boot
+
+    wait_for_app_to_be_up
+
+    logs = mrsk :app, :logs, capture: true
+    assert_match /App Host: vm1/, logs
+    assert_match /App Host: vm2/, logs
+    assert_match /GET \/ HTTP\/1.1/, logs
+
+    images = mrsk :app, :images, capture: true
+    assert_match /App Host: vm1/, images
+    assert_match /App Host: vm2/, images
+    assert_match /registry:4443\/app\s+#{latest_app_version}/, images
+    assert_match /registry:4443\/app\s+latest/, images
+
+    containers = mrsk :app, :containers, capture: true
+    assert_match /App Host: vm1/, containers
+    assert_match /App Host: vm2/, containers
+    assert_match /registry:4443\/app:#{latest_app_version}/, containers
+    assert_match /registry:4443\/app:latest/, containers
+
+    exec_output = mrsk :app, :exec, :ps, capture: true
+    assert_match /App Host: vm1/, exec_output
+    assert_match /App Host: vm2/, exec_output
+    assert_match /1 root      0:\d\d ps/, exec_output
+
+    exec_output = mrsk :app, :exec, "--reuse", :ps, capture: true
+    assert_match /App Host: vm1/, exec_output
+    assert_match /App Host: vm2/, exec_output
+    assert_match /1 root      0:\d\d nginx/, exec_output
+
+    mrsk :app, :remove
+
+    # traefik is up and returns 404s when it can't match a route
+    assert_app_not_found
+  end
+end

test/integration/docker-compose.yml

@@ -3,6 +3,8 @@ name: "mrsk-test"
 
 volumes:
   shared:
+  registry:
+  deployer_bundle:
 
 services:
   shared:
@@ -18,6 +20,8 @@ services:
     volumes:
       - ../..:/mrsk
       - shared:/shared
+      - registry:/registry
+      - deployer_bundle:/usr/local/bundle/
 
   registry:
     build:
@@ -28,6 +32,7 @@ services:
       - REGISTRY_HTTP_TLS_KEY=/certs/domain.key
     volumes:
       - shared:/shared
+      - registry:/var/lib/registry/
 
   vm1:
     privileged: true

test/integration/docker/deployer/app/.env.erb

@@ -0,0 +1 @@
+SECRET_TOKEN=1234

test/integration/docker/deployer/app/Dockerfile

@@ -1,4 +1,4 @@
-FROM nginx:1-alpine-slim
+FROM registry:4443/nginx:1-alpine-slim
 
 COPY default.conf /etc/nginx/conf.d/default.conf
 

test/integration/docker/deployer/app/config/deploy.yml

@@ -17,3 +17,11 @@ traefik:
   args:
     accesslog: true
     accesslog.format: json
+  image: registry:4443/traefik:v2.9
+accessories:
+  busybox:
+    image: registry:4443/busybox:1.36.0
+    cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done'
+    roles:
+      - web
+stop_wait_time: 1

test/integration/docker/deployer/boot.sh

@@ -1,9 +1,5 @@
 #!/bin/bash
 
-cd /mrsk && gem build mrsk.gemspec -o /tmp/mrsk.gem && gem install /tmp/mrsk.gem
-
 dockerd &
 
-trap "pkill -f sleep" term
+exec sleep infinity
-
-sleep infinity & wait

test/integration/docker/deployer/setup.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+install_mrsk() {
+  cd /mrsk && gem build mrsk.gemspec -o /tmp/mrsk.gem && gem install /tmp/mrsk.gem
+}
+
+# Push the images to a persistent volume on the registry container
+# This is to work around docker hub rate limits
+push_image_to_registry_4443() {
+  # Check if the image is in the registry without having to pull it
+  if ! stat /registry/docker/registry/v2/repositories/$1/_manifests/tags/$2/current/link > /dev/null; then
+    hub_tag=$1:$2
+    registry_4443_tag=registry:4443/$1:$2
+    docker pull $hub_tag
+    docker tag $hub_tag $registry_4443_tag
+    docker push $registry_4443_tag
+  fi
+}
+
+install_mrsk
+push_image_to_registry_4443 nginx 1-alpine-slim
+push_image_to_registry_4443 traefik v2.9
+push_image_to_registry_4443 busybox 1.36.0

test/integration/docker/load_balancer/default.conf

@@ -8,5 +8,9 @@ server {
 
   location / {
     proxy_pass http://loadbalancer;
+    proxy_connect_timeout       5;
+    proxy_send_timeout          5;
+    proxy_read_timeout          5;
+    send_timeout                5;
   }
 }

test/integration/docker/registry/boot.sh

@@ -2,6 +2,4 @@
 
 while [ ! -f /certs/domain.crt ]; do sleep 1; done
 
-trap "pkill -f registry" term
+exec /entrypoint.sh /etc/docker/registry/config.yml
-
-/entrypoint.sh /etc/docker/registry/config.yml & wait

test/integration/docker/shared/boot.sh

@@ -2,6 +2,4 @@
 
 cp -r * /shared
 
-trap "pkill -f sleep" term
+exec sleep infinity
-
-sleep infinity & wait

test/integration/docker/vm/boot.sh

@@ -6,6 +6,4 @@ service ssh restart
 
 dockerd &
 
-trap "pkill -f sleep" term
+exec sleep infinity
-
-sleep infinity & wait

test/integration/integration_test.rb

@@ -1,52 +1,19 @@
 require "net/http"
 require "test_helper"
 
-class DeployTest < ActiveSupport::TestCase
+class IntegrationTest < ActiveSupport::TestCase
-
   setup do
-    docker_compose "up --build --force-recreate -d"
+    docker_compose "up --build -d"
     wait_for_healthy
+    setup_deployer
   end
 
   teardown do
-    docker_compose "down -v"
+    docker_compose "down -t 1"
-  end
-
-  test "deploy" do
-    first_version = latest_app_version
-
-    assert_app_is_down
-
-    mrsk :deploy
-
-    assert_app_is_up version: first_version
-
-    second_version = update_app_rev
-
-    mrsk :redeploy
-
-    assert_app_is_up version: second_version
-
-    mrsk :rollback, first_version
-
-    assert_app_is_up version: first_version
-
-    details = mrsk :details, capture: true
-
-    assert_match /Traefik Host: vm1/, details
-    assert_match /Traefik Host: vm2/, details
-    assert_match /App Host: vm1/, details
-    assert_match /App Host: vm2/, details
-    assert_match /traefik:v2.9/, details
-    assert_match /registry:4443\/app:#{first_version}/, details
-
-    audit = mrsk :audit, capture: true
-
-    assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
   end
 
   private
-    def docker_compose(*commands, capture: false)
+    def docker_compose(*commands, capture: false, raise_on_error: true)
       command = "docker compose #{commands.join(" ")}"
       succeeded = false
       if capture
@@ -55,7 +22,7 @@ class DeployTest < ActiveSupport::TestCase
         succeeded = system("cd test/integration && #{command}")
       end
 
-      raise "Command `#{command}` failed with error code `#{$?}`" unless succeeded
+      raise "Command `#{command}` failed with error code `#{$?}`" if !succeeded && raise_on_error
       result
     end
 
@@ -68,24 +35,22 @@ class DeployTest < ActiveSupport::TestCase
     end
 
     def assert_app_is_down
-      assert_equal "502", app_response.code
+      response = app_response
+      debug_response_code(response, "502")
+      assert_equal "502", response.code
     end
 
     def assert_app_is_up(version: nil)
-      code = app_response.code
+      response = app_response
-      if code != "200"
+      debug_response_code(response, "200")
-        puts "Got response code #{code}, here are the traefik logs:"
+      assert_equal "200", response.code
-        mrsk :traefik, :logs
+      assert_app_version(version, response) if version
-        puts "And here are the load balancer logs"
-        docker_compose :logs, :load_balancer
-        puts "Tried to get the response code again and got #{app_response.code}"
-      end
-      assert_equal "200", code
-      assert_app_version(version) if version
     end
 
     def assert_app_not_found
-      assert_equal "404", app_response.code
+      response = app_response
+      debug_response_code(response, "404")
+      assert_equal "404", response.code
     end
 
     def wait_for_app_to_be_up(timeout: 10, up_count: 3)
@@ -101,7 +66,7 @@ class DeployTest < ActiveSupport::TestCase
     end
 
     def app_response
-      Net::HTTP.get_response(URI.parse("http://localhost:12345"))
+      Net::HTTP.get_response(URI.parse("http://localhost:12345/version"))
     end
 
     def update_app_rev
@@ -113,10 +78,8 @@ class DeployTest < ActiveSupport::TestCase
       deployer_exec("git rev-parse HEAD", capture: true)
     end
 
-    def assert_app_version(version)
+    def assert_app_version(version, response)
-      actual_version = Net::HTTP.get_response(URI.parse("http://localhost:12345/version")).body.strip
+      assert_equal version, response.body.strip
-
-      assert_equal version, actual_version
     end
 
     def wait_for_healthy(timeout: 20)
@@ -129,4 +92,20 @@ class DeployTest < ActiveSupport::TestCase
         sleep 0.1
       end
     end
+
+    def setup_deployer
+      deployer_exec("./setup.sh") unless $DEPLOYER_SETUP
+      $DEPLOYER_SETUP = true
+    end
+
+    def debug_response_code(app_response, expected_code)
+      code = app_response.code
+      if code != expected_code
+        puts "Got response code #{code}, here are the traefik logs:"
+        mrsk :traefik, :logs
+        puts "And here are the load balancer logs"
+        docker_compose :logs, :load_balancer
+        puts "Tried to get the response code again and got #{app_response.code}"
+      end
+    end
 end

test/integration/lock_test.rb

@@ -0,0 +1,18 @@
+require_relative "integration_test"
+
+class LockTest < IntegrationTest
+  test "acquire, release, status" do
+    mrsk :lock, :acquire, "-m 'Integration Tests'"
+
+    status = mrsk :lock, :status, capture: true
+    assert_match /Locked by: Deployer at .*\nVersion: #{latest_app_version}\nMessage: Integration Tests/m, status
+
+    error = mrsk :deploy, capture: true, raise_on_error: false
+    assert_match /Deploy lock found/m, error
+
+    mrsk :lock, :release
+
+    status = mrsk :lock, :status, capture: true
+    assert_match /There is no deploy lock/m, status
+  end
+end

test/integration/main_test.rb

@@ -0,0 +1,61 @@
+require_relative "integration_test"
+
+class MainTest < IntegrationTest
+  test "deploy, redeploy, rollback, details and audit" do
+    first_version = latest_app_version
+
+    assert_app_is_down
+
+    mrsk :deploy
+
+    assert_app_is_up version: first_version
+
+    second_version = update_app_rev
+
+    mrsk :redeploy
+
+    assert_app_is_up version: second_version
+
+    mrsk :rollback, first_version
+
+    assert_app_is_up version: first_version
+
+    details = mrsk :details, capture: true
+
+    assert_match /Traefik Host: vm1/, details
+    assert_match /Traefik Host: vm2/, details
+    assert_match /App Host: vm1/, details
+    assert_match /App Host: vm2/, details
+    assert_match /traefik:v2.9/, details
+    assert_match /registry:4443\/app:#{first_version}/, details
+
+    audit = mrsk :audit, capture: true
+
+    assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
+  end
+
+  test "envify" do
+    mrsk :envify
+
+    assert_equal "SECRET_TOKEN=1234", deployer_exec("cat .env", capture: true)
+  end
+
+  test "config" do
+    config = YAML.load(mrsk(:config, capture: true))
+    version = latest_app_version
+
+    assert_equal [ "web" ], config[:roles]
+    assert_equal [ "vm1", "vm2" ], config[:hosts]
+    assert_equal "vm1", config[:primary_host]
+    assert_equal version, config[:version]
+    assert_equal "registry:4443/app", config[:repository]
+    assert_equal "registry:4443/app:#{version}", config[:absolute_image]
+    assert_equal "app-#{version}", config[:service_with_version]
+    assert_equal [], config[:env_args]
+    assert_equal [], config[:volume_args]
+    assert_equal({ user: "root", auth_methods: [ "publickey" ] }, config[:ssh_options])
+    assert_equal({ "multiarch" => false, "args" => { "COMMIT_SHA" => version } }, config[:builder])
+    assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging]
+    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "cmd" => "wget -qO- http://localhost > /dev/null" }, config[:healthcheck])
+  end
+end

test/integration/traefik_test.rb

@@ -0,0 +1,36 @@
+require_relative "integration_test"
+
+class TraefikTest < IntegrationTest
+  test "boot, stop, start, restart, logs, remove" do
+    mrsk :traefik, :boot
+    assert_traefik_running
+
+    mrsk :traefik, :stop
+    assert_traefik_not_running
+
+    mrsk :traefik, :start
+    assert_traefik_running
+
+    mrsk :traefik, :restart
+    assert_traefik_running
+
+    logs = mrsk :traefik, :logs, capture: true
+    assert_match /Traefik version [\d.]+ built on/, logs
+
+    mrsk :traefik, :remove
+    assert_traefik_not_running
+  end
+
+  private
+    def assert_traefik_running
+      assert_match /traefik:v2.9   "\/entrypoint.sh/, traefik_details
+    end
+
+    def assert_traefik_not_running
+      refute_match /traefik:v2.9   "\/entrypoint.sh/, traefik_details
+    end
+
+    def traefik_details
+      mrsk :traefik, :details, capture: true
+    end
+end