From bcf8a927f54b78b75f7ff65858edfb8f27a894e5 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Wed, 12 Apr 2023 14:47:52 +0100 Subject: [PATCH] Run a mrsk deploy integration test Adds a simple integration test to ensure that `mrsk deploy` works. Everything required is spun up with docker compose: - shared: a container that contains an ssh key and a self signed cert to be shared between the images - deployer: the image we will deploy from - registry: a docker registry - two vm images to deploy into - load_balancer: an nginx load balancer to use between our images The other images are in privileged mode so that we can run docker-in-docker. We need to run docker inside the images - mapping in the docker socket doesn't work because both VMs would share the host daemon. The docker registry requires a self signed cert as you cannot use basic auth over HTTP except on localhost. It runs on port 4443 rather than 443 because docker refused to accept that "registry" is a docker host and tries to push images to docker.io/registry. "registry:4443" works fine. The shared container contains the ssh keys for the deployer and vms, and the self signed cert for the registry. When the shared container boots, it copies them into a shared volume. The other deployer and vm images are built with soft links from the shared volume to the require locations. Their boot scripts wait for the files to be copied in before continuing. The root mrsk folder is mapped into the deployer container. On boot it builds the gem and installs it. Right now there's just a single test. We confirm that the load balancer is returning a 502, run `mrsk deploy` and then confirm it returns 200. --- test/integration/deploy_test.rb | 42 ++++++++++++++++ test/integration/docker-compose.yml | 50 +++++++++++++++++++ test/integration/docker/deployer/Dockerfile | 27 ++++++++++ .../docker/deployer/app/Dockerfile | 3 ++ .../docker/deployer/app/config/deploy.yml | 14 ++++++ .../docker/deployer/app/default.conf | 17 +++++++ test/integration/docker/deployer/boot.sh | 9 ++++ .../docker/load_balancer/Dockerfile | 4 ++ .../docker/load_balancer/default.conf | 12 +++++ test/integration/docker/registry/Dockerfile | 7 +++ test/integration/docker/registry/boot.sh | 7 +++ test/integration/docker/shared/Dockerfile | 14 ++++++ .../docker/shared/registry-dns.conf | 7 +++ test/integration/docker/vm/Dockerfile | 12 +++++ test/integration/docker/vm/boot.sh | 11 ++++ 15 files changed, 236 insertions(+) create mode 100644 test/integration/deploy_test.rb create mode 100644 test/integration/docker-compose.yml create mode 100644 test/integration/docker/deployer/Dockerfile create mode 100644 test/integration/docker/deployer/app/Dockerfile create mode 100644 test/integration/docker/deployer/app/config/deploy.yml create mode 100644 test/integration/docker/deployer/app/default.conf create mode 100755 test/integration/docker/deployer/boot.sh create mode 100644 test/integration/docker/load_balancer/Dockerfile create mode 100644 test/integration/docker/load_balancer/default.conf create mode 100644 test/integration/docker/registry/Dockerfile create mode 100755 test/integration/docker/registry/boot.sh create mode 100644 test/integration/docker/shared/Dockerfile create mode 100644 test/integration/docker/shared/registry-dns.conf create mode 100644 test/integration/docker/vm/Dockerfile create mode 100755 test/integration/docker/vm/boot.sh diff --git a/test/integration/deploy_test.rb b/test/integration/deploy_test.rb new file mode 100644 index 00000000..f696cb15 --- /dev/null +++ b/test/integration/deploy_test.rb @@ -0,0 +1,42 @@ +require "net/http" + +class DeployTest < ActiveSupport::TestCase + + setup do + docker_compose "up --build --force-recreate -d" + sleep 5 + end + + teardown do + docker_compose "down -v" + end + + test "deploy" do + assert_app_is_down + + mrsk :deploy + + assert_app_is_up + end + + private + def docker_compose(*commands) + system("cd test/integration && docker compose #{commands.join(" ")}") + end + + def mrsk(*commands) + docker_compose("exec deployer mrsk #{commands.join(" ")}") + end + + def assert_app_is_down + assert_equal "502", app_response.code + end + + def assert_app_is_up + assert_equal "200", app_response.code + end + + def app_response + Net::HTTP.get_response(URI.parse("http://localhost:12345")) + end +end diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml new file mode 100644 index 00000000..c9fdf306 --- /dev/null +++ b/test/integration/docker-compose.yml @@ -0,0 +1,50 @@ +version: "3.7" +name: "mrsk-test" + +volumes: + shared: + +services: + shared: + build: + context: docker/shared + volumes: + - shared:/shared + + deployer: + privileged: true + build: + context: docker/deployer + volumes: + - ../..:/mrsk + - shared:/shared + + registry: + build: + context: docker/registry + environment: + - REGISTRY_HTTP_ADDR=0.0.0.0:4443 + - REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt + - REGISTRY_HTTP_TLS_KEY=/certs/domain.key + volumes: + - shared:/shared + + vm1: + privileged: true + build: + context: docker/vm + volumes: + - shared:/shared + + vm2: + privileged: true + build: + context: docker/vm + volumes: + - shared:/shared + + load_balancer: + build: + context: docker/load_balancer + ports: + - "12345:80" diff --git a/test/integration/docker/deployer/Dockerfile b/test/integration/docker/deployer/Dockerfile new file mode 100644 index 00000000..fbfc77c4 --- /dev/null +++ b/test/integration/docker/deployer/Dockerfile @@ -0,0 +1,27 @@ +FROM ruby:3.2 + +WORKDIR /app + +RUN apt-get update && apt-get install -y ca-certificates openssh-client curl gnupg docker.io + +RUN install -m 0755 -d /etc/apt/keyrings +RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg +RUN chmod a+r /etc/apt/keyrings/docker.gpg +RUN echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null + +RUN apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +COPY boot.sh . +COPY app/ . + +RUN ln -s /shared/ssh /root/.ssh +RUN mkdir -p /etc/docker/certs.d/registry:4443 && ln -s /shared/certs/domain.crt /etc/docker/certs.d/registry:4443/ca.crt + +RUN git config --global user.email "deployer@example.com" +RUN git config --global user.name "Deployer" +RUN git init && git add . && git commit -am "Initial version" + +CMD ["./boot.sh"] diff --git a/test/integration/docker/deployer/app/Dockerfile b/test/integration/docker/deployer/app/Dockerfile new file mode 100644 index 00000000..b173cc99 --- /dev/null +++ b/test/integration/docker/deployer/app/Dockerfile @@ -0,0 +1,3 @@ +FROM nginx:1-alpine-slim + +COPY default.conf /etc/nginx/conf.d/default.conf diff --git a/test/integration/docker/deployer/app/config/deploy.yml b/test/integration/docker/deployer/app/config/deploy.yml new file mode 100644 index 00000000..5ac25b14 --- /dev/null +++ b/test/integration/docker/deployer/app/config/deploy.yml @@ -0,0 +1,14 @@ +service: app +image: app +servers: + - vm1 + - vm2 +registry: + server: registry:4443 + username: root + password: root +builder: + multiarch: false +healthcheck: + path: / + port: 80 diff --git a/test/integration/docker/deployer/app/default.conf b/test/integration/docker/deployer/app/default.conf new file mode 100644 index 00000000..e37a9bc1 --- /dev/null +++ b/test/integration/docker/deployer/app/default.conf @@ -0,0 +1,17 @@ +server { + listen 80; + listen [::]:80; + server_name localhost; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} diff --git a/test/integration/docker/deployer/boot.sh b/test/integration/docker/deployer/boot.sh new file mode 100755 index 00000000..ac42c53a --- /dev/null +++ b/test/integration/docker/deployer/boot.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +cd /mrsk && gem build mrsk.gemspec -o /tmp/mrsk.gem && gem install /tmp/mrsk.gem + +dockerd & + +trap "pkill -f sleep" term + +sleep infinity & wait diff --git a/test/integration/docker/load_balancer/Dockerfile b/test/integration/docker/load_balancer/Dockerfile new file mode 100644 index 00000000..eec81a54 --- /dev/null +++ b/test/integration/docker/load_balancer/Dockerfile @@ -0,0 +1,4 @@ +FROM nginx:1-alpine-slim + +COPY default.conf /etc/nginx/conf.d/default.conf + diff --git a/test/integration/docker/load_balancer/default.conf b/test/integration/docker/load_balancer/default.conf new file mode 100644 index 00000000..3b734e36 --- /dev/null +++ b/test/integration/docker/load_balancer/default.conf @@ -0,0 +1,12 @@ +upstream loadbalancer { + server vm1:80; + server vm2:80; +} + +server { + listen 80; + + location / { + proxy_pass http://loadbalancer; + } +} diff --git a/test/integration/docker/registry/Dockerfile b/test/integration/docker/registry/Dockerfile new file mode 100644 index 00000000..87d2c62b --- /dev/null +++ b/test/integration/docker/registry/Dockerfile @@ -0,0 +1,7 @@ +FROM registry + +COPY boot.sh . + +RUN ln -s /shared/certs /certs + +ENTRYPOINT ["./boot.sh"] diff --git a/test/integration/docker/registry/boot.sh b/test/integration/docker/registry/boot.sh new file mode 100755 index 00000000..411bc617 --- /dev/null +++ b/test/integration/docker/registry/boot.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +while [ ! -f /certs/domain.crt ]; do sleep 1; done + +trap "pkill -f registry" term + +/entrypoint.sh /etc/docker/registry/config.yml & wait diff --git a/test/integration/docker/shared/Dockerfile b/test/integration/docker/shared/Dockerfile new file mode 100644 index 00000000..0cd5aa60 --- /dev/null +++ b/test/integration/docker/shared/Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:22.10 + +WORKDIR /work + +RUN apt-get update && apt-get -y install openssh-client openssl + +RUN mkdir ssh && \ + ssh-keygen -t rsa -f ssh/id_rsa -N "" + +COPY registry-dns.conf . + +RUN mkdir certs && openssl req -newkey rsa:4096 -nodes -sha256 -keyout certs/domain.key -x509 -days 365 -out certs/domain.crt -subj '/CN=registry' -extensions EXT -config registry-dns.conf + +CMD ["bash", "-c", "cp -r * /shared"] diff --git a/test/integration/docker/shared/registry-dns.conf b/test/integration/docker/shared/registry-dns.conf new file mode 100644 index 00000000..eff1c7bd --- /dev/null +++ b/test/integration/docker/shared/registry-dns.conf @@ -0,0 +1,7 @@ +[dn] +CN=registry +[req] +distinguished_name = dn +[EXT] +subjectAltName=DNS:registry +keyUsage=digitalSignature diff --git a/test/integration/docker/vm/Dockerfile b/test/integration/docker/vm/Dockerfile new file mode 100644 index 00000000..1fcb9278 --- /dev/null +++ b/test/integration/docker/vm/Dockerfile @@ -0,0 +1,12 @@ +FROM ubuntu:22.10 + +WORKDIR /work + +RUN apt-get update && apt-get -y install openssh-client openssh-server docker.io + +RUN mkdir /root/.ssh && ln -s /shared/ssh/id_rsa.pub /root/.ssh/authorized_keys +RUN mkdir -p /etc/docker/certs.d/registry:4443 && ln -s /shared/certs/domain.crt /etc/docker/certs.d/registry:4443/ca.crt + +COPY boot.sh . + +CMD ["./boot.sh"] diff --git a/test/integration/docker/vm/boot.sh b/test/integration/docker/vm/boot.sh new file mode 100755 index 00000000..5a26ab2e --- /dev/null +++ b/test/integration/docker/vm/boot.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +while [ ! -f /root/.ssh/authorized_keys ]; do echo "Waiting for ssh keys"; sleep 1; done + +service ssh restart + +dockerd & + +trap "pkill -f sleep" term + +sleep infinity & wait