From 52ca5b846aa6a91a37bf7c60fd8cf9179d99fe23 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Tue, 25 Apr 2023 15:32:48 +0100
Subject: [PATCH] Wait for healthy containers in integration test

Rather than waiting 5 seconds and hoping for the best after we boot
docker compose, add docker healthchecks and wait for all the containers
to be healthy.
---
 test/cli/cli_test_case.rb                     | 12 +------
 test/integration/deploy_test.rb               | 36 ++++++++++++++++---
 test/integration/docker/deployer/Dockerfile   |  2 ++
 .../docker/load_balancer/Dockerfile           |  1 +
 test/integration/docker/registry/Dockerfile   |  2 ++
 test/integration/docker/shared/Dockerfile     |  5 ++-
 test/integration/docker/shared/boot.sh        |  7 ++++
 test/integration/docker/vm/Dockerfile         |  2 ++
 test/test_helper.rb                           | 11 ++++++
 9 files changed, 61 insertions(+), 17 deletions(-)
 create mode 100755 test/integration/docker/shared/boot.sh

diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb
index 01cf0019..abd805ec 100644
--- a/test/cli/cli_test_case.rb
+++ b/test/cli/cli_test_case.rb
@@ -1,5 +1,4 @@
 require "test_helper"
-require "active_support/testing/stream"
 
 class CliTestCase < ActiveSupport::TestCase
   include ActiveSupport::Testing::Stream
@@ -17,13 +16,4 @@ class CliTestCase < ActiveSupport::TestCase
     ENV.delete("MYSQL_ROOT_PASSWORD")
     ENV.delete("VERSION")
   end
-
-  private
-    def stdouted
-      capture(:stdout) { yield }.strip
-    end
-
-    def stderred
-      capture(:stderr) { yield }.strip
-    end
-  end
+end
diff --git a/test/integration/deploy_test.rb b/test/integration/deploy_test.rb
index f696cb15..89b2269f 100644
--- a/test/integration/deploy_test.rb
+++ b/test/integration/deploy_test.rb
@@ -1,10 +1,11 @@
 require "net/http"
+require "test_helper"
 
 class DeployTest < ActiveSupport::TestCase
 
   setup do
     docker_compose "up --build --force-recreate -d"
-    sleep 5
+    wait_for_healthy
   end
 
   teardown do
@@ -20,12 +21,29 @@ class DeployTest < ActiveSupport::TestCase
   end
 
   private
-    def docker_compose(*commands)
-      system("cd test/integration && docker compose #{commands.join(" ")}")
+    def docker_compose(*commands, capture: false)
+      command = "docker compose #{commands.join(" ")}"
+      succeeded = false
+      if capture
+        result = stdouted { succeeded = system("cd test/integration && #{command}") }
+      else
+        succeeded = system("cd test/integration && #{command}")
+      end
+
+      raise "Command `#{command}` failed with error code `#{$?}`" unless succeeded
+      result
     end
 
-    def mrsk(*commands)
-      docker_compose("exec deployer mrsk #{commands.join(" ")}")
+    def deployer_exec(*commands, capture: false)
+      if capture
+        stdouted { docker_compose("exec deployer #{commands.join(" ")}") }
+      else
+        docker_compose("exec deployer #{commands.join(" ")}", capture: capture)
+      end
+    end
+
+    def mrsk(*commands, capture: false)
+      deployer_exec(:mrsk, *commands, capture: capture)
     end
 
     def assert_app_is_down
@@ -39,4 +57,12 @@ class DeployTest < ActiveSupport::TestCase
     def app_response
       Net::HTTP.get_response(URI.parse("http://localhost:12345"))
     end
+
+    def wait_for_healthy(timeout: 20)
+      timeout_at = Time.now + timeout
+      while docker_compose("ps -a | tail -n +2 | grep -v '(healthy)' | wc -l", capture: true) != "0"
+        raise "Container not healthy after #{timeout} seconds" if timeout_at < Time.now
+        sleep 0.1
+      end
+    end
 end
diff --git a/test/integration/docker/deployer/Dockerfile b/test/integration/docker/deployer/Dockerfile
index fbfc77c4..22556cc2 100644
--- a/test/integration/docker/deployer/Dockerfile
+++ b/test/integration/docker/deployer/Dockerfile
@@ -24,4 +24,6 @@ RUN git config --global user.email "deployer@example.com"
 RUN git config --global user.name "Deployer"
 RUN git init && git add . && git commit -am "Initial version"
 
+HEALTHCHECK --interval=1s CMD pgrep sleep
+
 CMD ["./boot.sh"]
diff --git a/test/integration/docker/load_balancer/Dockerfile b/test/integration/docker/load_balancer/Dockerfile
index eec81a54..d70273ce 100644
--- a/test/integration/docker/load_balancer/Dockerfile
+++ b/test/integration/docker/load_balancer/Dockerfile
@@ -2,3 +2,4 @@ FROM nginx:1-alpine-slim
 
 COPY default.conf /etc/nginx/conf.d/default.conf
 
+HEALTHCHECK --interval=1s CMD pgrep nginx
diff --git a/test/integration/docker/registry/Dockerfile b/test/integration/docker/registry/Dockerfile
index 87d2c62b..f5eefc33 100644
--- a/test/integration/docker/registry/Dockerfile
+++ b/test/integration/docker/registry/Dockerfile
@@ -4,4 +4,6 @@ COPY boot.sh .
 
 RUN ln -s /shared/certs /certs
 
+HEALTHCHECK --interval=1s CMD pgrep registry
+
 ENTRYPOINT ["./boot.sh"]
diff --git a/test/integration/docker/shared/Dockerfile b/test/integration/docker/shared/Dockerfile
index 0cd5aa60..dae69053 100644
--- a/test/integration/docker/shared/Dockerfile
+++ b/test/integration/docker/shared/Dockerfile
@@ -8,7 +8,10 @@ RUN mkdir ssh && \
   ssh-keygen -t rsa -f ssh/id_rsa -N ""
 
 COPY registry-dns.conf .
+COPY boot.sh .
 
 RUN mkdir certs && openssl req -newkey rsa:4096 -nodes -sha256 -keyout certs/domain.key   -x509 -days 365 -out certs/domain.crt   -subj '/CN=registry' -extensions EXT -config registry-dns.conf
 
-CMD ["bash", "-c", "cp -r * /shared"]
+HEALTHCHECK --interval=1s CMD pgrep sleep
+
+CMD ["./boot.sh"]
diff --git a/test/integration/docker/shared/boot.sh b/test/integration/docker/shared/boot.sh
new file mode 100755
index 00000000..821c8c30
--- /dev/null
+++ b/test/integration/docker/shared/boot.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cp -r * /shared
+
+trap "pkill -f sleep" term
+
+sleep infinity & wait
diff --git a/test/integration/docker/vm/Dockerfile b/test/integration/docker/vm/Dockerfile
index 1fcb9278..99f881fa 100644
--- a/test/integration/docker/vm/Dockerfile
+++ b/test/integration/docker/vm/Dockerfile
@@ -9,4 +9,6 @@ RUN mkdir -p /etc/docker/certs.d/registry:4443 && ln -s /shared/certs/domain.crt
 
 COPY boot.sh .
 
+HEALTHCHECK --interval=1s CMD pgrep dockerd
+
 CMD ["./boot.sh"]
diff --git a/test/test_helper.rb b/test/test_helper.rb
index 3704e8e2..f23f0a92 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -1,6 +1,7 @@
 require "bundler/setup"
 require "active_support/test_case"
 require "active_support/testing/autorun"
+require "active_support/testing/stream"
 require "debug"
 require "mocha/minitest" # using #stubs that can alter returns
 require "minitest/autorun" # using #stub that take args
@@ -23,4 +24,14 @@ module SSHKit
 end
 
 class ActiveSupport::TestCase
+  include ActiveSupport::Testing::Stream
+
+  private
+    def stdouted
+      capture(:stdout) { yield }.strip
+    end
+
+    def stderred
+      capture(:stderr) { yield }.strip
+    end
 end