From a77428143fbbdbd5b8000aa7411eb34728a7fc09 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 28 Apr 2023 14:19:15 +0100 Subject: [PATCH 1/8] Fix the integration test healthcheck The alpine nginx container doesn't contain curl, so let's override the healthcheck command to use wget. --- test/integration/docker/deployer/app/config/deploy.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/integration/docker/deployer/app/config/deploy.yml b/test/integration/docker/deployer/app/config/deploy.yml index 5ac25b14..bd97dc36 100644 --- a/test/integration/docker/deployer/app/config/deploy.yml +++ b/test/integration/docker/deployer/app/config/deploy.yml @@ -10,5 +10,4 @@ registry: builder: multiarch: false healthcheck: - path: / - port: 80 + cmd: wget -qO- http://localhost > /dev/null From 494a1ae089c66fe183ee5a1f4f38e5d725063e77 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Mon, 1 May 2023 12:11:15 +0100 Subject: [PATCH 2/8] Report on container health after failure --- test/integration/deploy_test.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/integration/deploy_test.rb b/test/integration/deploy_test.rb index 89b2269f..0fb94d27 100644 --- a/test/integration/deploy_test.rb +++ b/test/integration/deploy_test.rb @@ -61,7 +61,10 @@ class DeployTest < ActiveSupport::TestCase def wait_for_healthy(timeout: 20) timeout_at = Time.now + timeout while docker_compose("ps -a | tail -n +2 | grep -v '(healthy)' | wc -l", capture: true) != "0" - raise "Container not healthy after #{timeout} seconds" if timeout_at < Time.now + if timeout_at < Time.now + docker_compose("ps -a | tail -n +2 | grep -v '(healthy)'") + raise "Container not healthy after #{timeout} seconds" if timeout_at < Time.now + end sleep 0.1 end end From ca2e2bac2ecaa73b97fafe455c9a2ed555b3c4b2 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Mon, 1 May 2023 12:50:45 +0100 Subject: [PATCH 3/8] Fix missing for apt-get --- test/integration/docker/deployer/Dockerfile | 4 ++-- test/integration/docker/shared/Dockerfile | 2 +- test/integration/docker/vm/Dockerfile | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/docker/deployer/Dockerfile b/test/integration/docker/deployer/Dockerfile index 22556cc2..ccd1de39 100644 --- a/test/integration/docker/deployer/Dockerfile +++ b/test/integration/docker/deployer/Dockerfile @@ -2,7 +2,7 @@ FROM ruby:3.2 WORKDIR /app -RUN apt-get update && apt-get install -y ca-certificates openssh-client curl gnupg docker.io +RUN apt-get update --fix-missing && apt-get install -y ca-certificates openssh-client curl gnupg docker.io RUN install -m 0755 -d /etc/apt/keyrings RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg @@ -12,7 +12,7 @@ RUN echo \ "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ tee /etc/apt/sources.list.d/docker.list > /dev/null -RUN apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +RUN apt-get update --fix-missing && apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin COPY boot.sh . COPY app/ . diff --git a/test/integration/docker/shared/Dockerfile b/test/integration/docker/shared/Dockerfile index dae69053..bc0d8e84 100644 --- a/test/integration/docker/shared/Dockerfile +++ b/test/integration/docker/shared/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:22.10 WORKDIR /work -RUN apt-get update && apt-get -y install openssh-client openssl +RUN apt-get update --fix-missing && apt-get -y install openssh-client openssl RUN mkdir ssh && \ ssh-keygen -t rsa -f ssh/id_rsa -N "" diff --git a/test/integration/docker/vm/Dockerfile b/test/integration/docker/vm/Dockerfile index 99f881fa..f481023c 100644 --- a/test/integration/docker/vm/Dockerfile +++ b/test/integration/docker/vm/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:22.10 WORKDIR /work -RUN apt-get update && apt-get -y install openssh-client openssh-server docker.io +RUN apt-get update --fix-missing && apt-get -y install openssh-client openssh-server docker.io RUN mkdir /root/.ssh && ln -s /shared/ssh/id_rsa.pub /root/.ssh/authorized_keys RUN mkdir -p /etc/docker/certs.d/registry:4443 && ln -s /shared/certs/domain.crt /etc/docker/certs.d/registry:4443/ca.crt From 548a1019c14785f572710b94c3bc4fea7b9212cd Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Mon, 1 May 2023 18:21:22 +0100 Subject: [PATCH 4/8] Dump traefik logs when app not booted --- test/integration/deploy_test.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/integration/deploy_test.rb b/test/integration/deploy_test.rb index 0fb94d27..1661b23a 100644 --- a/test/integration/deploy_test.rb +++ b/test/integration/deploy_test.rb @@ -51,7 +51,12 @@ class DeployTest < ActiveSupport::TestCase end def assert_app_is_up - assert_equal "200", app_response.code + code = app_response.code + if code != "200" + puts "Got response code #{code}, here are the traefik logs:" + mrsk :traefik, :logs + end + assert_equal "200", code end def app_response From 94f87eddedfd88f011e10128602de3ddaea3727b Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Mon, 1 May 2023 18:27:08 +0100 Subject: [PATCH 5/8] Also dump load balancer logs --- test/integration/deploy_test.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/deploy_test.rb b/test/integration/deploy_test.rb index 1661b23a..ec894caf 100644 --- a/test/integration/deploy_test.rb +++ b/test/integration/deploy_test.rb @@ -55,6 +55,8 @@ class DeployTest < ActiveSupport::TestCase if code != "200" puts "Got response code #{code}, here are the traefik logs:" mrsk :traefik, :logs + puts "Add here are the load balancer logs" + docker_compose :logs, :load_balancer end assert_equal "200", code end From 1170e2311eef53fe6433071f8d418381a16dee98 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Mon, 1 May 2023 18:32:07 +0100 Subject: [PATCH 6/8] Check if we are still getting a 404 --- test/integration/deploy_test.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/deploy_test.rb b/test/integration/deploy_test.rb index ec894caf..351dc5d2 100644 --- a/test/integration/deploy_test.rb +++ b/test/integration/deploy_test.rb @@ -55,8 +55,9 @@ class DeployTest < ActiveSupport::TestCase if code != "200" puts "Got response code #{code}, here are the traefik logs:" mrsk :traefik, :logs - puts "Add here are the load balancer logs" + puts "And here are the load balancer logs" docker_compose :logs, :load_balancer + puts "Tried to get the response code again and got #{app_response.code}" end assert_equal "200", code end From 650f9b1fbf7596213309b8ba6e4eb8c7fd5bb600 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Mon, 1 May 2023 18:55:10 +0100 Subject: [PATCH 7/8] Include traefik access logs --- test/integration/docker/deployer/app/config/deploy.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/integration/docker/deployer/app/config/deploy.yml b/test/integration/docker/deployer/app/config/deploy.yml index bd97dc36..8442c57b 100644 --- a/test/integration/docker/deployer/app/config/deploy.yml +++ b/test/integration/docker/deployer/app/config/deploy.yml @@ -11,3 +11,7 @@ builder: multiarch: false healthcheck: cmd: wget -qO- http://localhost > /dev/null +traefik: + args: + accesslog: true + accesslog.format: json From d0f66db33c7de2004b601deb796f24cb97fe871e Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Mon, 1 May 2023 18:58:46 +0100 Subject: [PATCH 8/8] Extend traefik delay by 1 second --- lib/mrsk/utils/healthcheck_poller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mrsk/utils/healthcheck_poller.rb b/lib/mrsk/utils/healthcheck_poller.rb index d7b8be65..3ef5b7a8 100644 --- a/lib/mrsk/utils/healthcheck_poller.rb +++ b/lib/mrsk/utils/healthcheck_poller.rb @@ -1,5 +1,5 @@ class Mrsk::Utils::HealthcheckPoller - TRAEFIK_HEALTHY_DELAY = 1 + TRAEFIK_HEALTHY_DELAY = 2 class HealthcheckError < StandardError; end