Wait for healthy containers in integration test

Rather than waiting 5 seconds and hoping for the best after we boot
docker compose, add docker healthchecks and wait for all the containers
to be healthy.

test/cli/cli_test_case.rb

@@ -1,5 +1,4 @@
 require "test_helper"
-require "active_support/testing/stream"
 
 class CliTestCase < ActiveSupport::TestCase
   include ActiveSupport::Testing::Stream
@@ -17,13 +16,4 @@ class CliTestCase < ActiveSupport::TestCase
     ENV.delete("MYSQL_ROOT_PASSWORD")
     ENV.delete("VERSION")
   end
-
+end
-  private
-    def stdouted
-      capture(:stdout) { yield }.strip
-    end
-
-    def stderred
-      capture(:stderr) { yield }.strip
-    end
-  end

test/integration/deploy_test.rb

@@ -1,10 +1,11 @@
 require "net/http"
+require "test_helper"
 
 class DeployTest < ActiveSupport::TestCase
 
   setup do
     docker_compose "up --build --force-recreate -d"
-    sleep 5
+    wait_for_healthy
   end
 
   teardown do
@@ -20,12 +21,29 @@ class DeployTest < ActiveSupport::TestCase
   end
 
   private
-    def docker_compose(*commands)
+    def docker_compose(*commands, capture: false)
-      system("cd test/integration && docker compose #{commands.join(" ")}")
+      command = "docker compose #{commands.join(" ")}"
+      succeeded = false
+      if capture
+        result = stdouted { succeeded = system("cd test/integration && #{command}") }
+      else
+        succeeded = system("cd test/integration && #{command}")
+      end
+
+      raise "Command `#{command}` failed with error code `#{$?}`" unless succeeded
+      result
     end
 
-    def mrsk(*commands)
+    def deployer_exec(*commands, capture: false)
-      docker_compose("exec deployer mrsk #{commands.join(" ")}")
+      if capture
+        stdouted { docker_compose("exec deployer #{commands.join(" ")}") }
+      else
+        docker_compose("exec deployer #{commands.join(" ")}", capture: capture)
+      end
+    end
+
+    def mrsk(*commands, capture: false)
+      deployer_exec(:mrsk, *commands, capture: capture)
     end
 
     def assert_app_is_down
@@ -39,4 +57,12 @@ class DeployTest < ActiveSupport::TestCase
     def app_response
       Net::HTTP.get_response(URI.parse("http://localhost:12345"))
     end
+
+    def wait_for_healthy(timeout: 20)
+      timeout_at = Time.now + timeout
+      while docker_compose("ps -a | tail -n +2 | grep -v '(healthy)' | wc -l", capture: true) != "0"
+        raise "Container not healthy after #{timeout} seconds" if timeout_at < Time.now
+        sleep 0.1
+      end
+    end
 end

test/integration/docker/deployer/Dockerfile

@@ -24,4 +24,6 @@ RUN git config --global user.email "deployer@example.com"
 RUN git config --global user.name "Deployer"
 RUN git init && git add . && git commit -am "Initial version"
 
+HEALTHCHECK --interval=1s CMD pgrep sleep
+
 CMD ["./boot.sh"]

test/integration/docker/load_balancer/Dockerfile

@@ -2,3 +2,4 @@ FROM nginx:1-alpine-slim
 
 COPY default.conf /etc/nginx/conf.d/default.conf
 
+HEALTHCHECK --interval=1s CMD pgrep nginx

test/integration/docker/registry/Dockerfile

@@ -4,4 +4,6 @@ COPY boot.sh .
 
 RUN ln -s /shared/certs /certs
 
+HEALTHCHECK --interval=1s CMD pgrep registry
+
 ENTRYPOINT ["./boot.sh"]

test/integration/docker/shared/Dockerfile

@@ -8,7 +8,10 @@ RUN mkdir ssh && \
   ssh-keygen -t rsa -f ssh/id_rsa -N ""
 
 COPY registry-dns.conf .
+COPY boot.sh .
 
 RUN mkdir certs && openssl req -newkey rsa:4096 -nodes -sha256 -keyout certs/domain.key   -x509 -days 365 -out certs/domain.crt   -subj '/CN=registry' -extensions EXT -config registry-dns.conf
 
-CMD ["bash", "-c", "cp -r * /shared"]
+HEALTHCHECK --interval=1s CMD pgrep sleep
+
+CMD ["./boot.sh"]

test/integration/docker/shared/boot.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cp -r * /shared
+
+trap "pkill -f sleep" term
+
+sleep infinity & wait

test/integration/docker/vm/Dockerfile

@@ -9,4 +9,6 @@ RUN mkdir -p /etc/docker/certs.d/registry:4443 && ln -s /shared/certs/domain.crt
 
 COPY boot.sh .
 
+HEALTHCHECK --interval=1s CMD pgrep dockerd
+
 CMD ["./boot.sh"]

test/test_helper.rb

@@ -1,6 +1,7 @@
 require "bundler/setup"
 require "active_support/test_case"
 require "active_support/testing/autorun"
+require "active_support/testing/stream"
 require "debug"
 require "mocha/minitest" # using #stubs that can alter returns
 require "minitest/autorun" # using #stub that take args
@@ -23,4 +24,14 @@ module SSHKit
 end
 
 class ActiveSupport::TestCase
+  include ActiveSupport::Testing::Stream
+
+  private
+    def stdouted
+      capture(:stdout) { yield }.strip
+    end
+
+    def stderred
+      capture(:stderr) { yield }.strip
+    end
 end