Add kamal-proxy in experimental mode

The proxy can be enabled via the config:

```
proxy:
  enabled: true
  hosts:
    - 10.0.0.1
    - 10.0.0.2
```

This will enable the proxy and cause it to be run on the hosts listed
under `hosts`, after running `kamal proxy reboot`.

Enabling the proxy disables `kamal traefik` commands and replaces them
with `kamal proxy` ones. However only the marked hosts will run the
kamal-proxy container, the rest will run Traefik as before.

test/cli/app_test.rb

@@ -356,6 +356,18 @@ class CliAppTest < CliTestCase
     end
   end
 
+  test "boot proxy" do
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version
+
+    run_command("boot", config: :with_proxy).tap do |output|
+      assert_match /Renaming container .* to .* as already deployed on 1.1.1.1/, output # Rename
+      assert_match /docker rename app-web-latest app-web-latest_replaced_[0-9a-f]{16}/, output
+      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} -e KAMAL_CONTAINER_NAME="app-web-latest" -e KAMAL_VERSION="latest" --env-file .kamal\/env\/roles\/app-web.env --log-opt max-size="10m" --label service="app" --label role="web" --label destination dhh\/app:latest/, output
+      assert_match /docker exec kamal-proxy kamal-proxy deploy app-web --target "123"/, output
+      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+    end
+  end
+
   private
     def run_command(*command, config: :with_accessories, host: "1.1.1.1", allow_execute_error: false)
       stdouted do

test/cli/proxy_test.rb

@@ -0,0 +1,141 @@
+require_relative "cli_test_case"
+
+class CliProxyTest < CliTestCase
+  test "boot" do
+    run_command("boot").tap do |output|
+      assert_match "docker login", output
+      assert_match "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}", output
+    end
+  end
+
+  test "reboot" do
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'", "|", :sed, "-e", "'s/\\/tcp$//'")
+      .returns("172.1.0.2:80")
+      .at_least_once
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with { |*args| args[0..1] == [ :sh, "-c" ] }
+      .returns("123")
+      .at_least_once
+
+    run_command("reboot", "-y").tap do |output|
+      assert_match "docker container stop kamal-proxy on 1.1.1.1", output
+      assert_match "docker container stop traefik on 1.1.1.1", output
+      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.1", output
+      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.1", output
+      assert_match "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE} on 1.1.1.1", output
+      assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target \"172.1.0.2:80\" --deploy-timeout \"6s\" on 1.1.1.1", output
+
+      assert_match "docker container stop kamal-proxy on 1.1.1.2", output
+      assert_match "docker container stop traefik on 1.1.1.2", output
+      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.2", output
+      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.2", output
+      assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" traefik:v2.10 --providers.docker --log.level=\"DEBUG\" on 1.1.1.2", output
+    end
+  end
+
+  test "reboot --rolling" do
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'", "|", :sed, "-e", "'s/\\/tcp$//'")
+      .returns("172.1.0.2:80")
+      .at_least_once
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with { |*args| args[0..1] == [ :sh, "-c" ] }
+      .returns("123")
+      .at_least_once
+
+    run_command("reboot", "--rolling", "-y").tap do |output|
+      assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.1", output
+    end
+  end
+
+  test "start" do
+    run_command("start").tap do |output|
+      assert_match "docker container start kamal-proxy", output
+    end
+  end
+
+  test "stop" do
+    run_command("stop").tap do |output|
+      assert_match "docker container stop kamal-proxy", output
+    end
+  end
+
+  test "restart" do
+    Kamal::Cli::Proxy.any_instance.expects(:stop)
+    Kamal::Cli::Proxy.any_instance.expects(:start)
+
+    run_command("restart")
+  end
+
+  test "details" do
+    run_command("details").tap do |output|
+      assert_match "docker ps --filter name=^kamal-proxy$", output
+    end
+  end
+
+  test "logs" do
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture)
+      .with(:docker, :logs, "kamal-proxy", " --tail 100", "--timestamps", "2>&1")
+      .returns("Log entry")
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture)
+      .with(:docker, :logs, "traefik", " --tail 100", "--timestamps", "2>&1")
+      .returns("Log entry")
+
+    run_command("logs").tap do |output|
+      assert_match "Proxy Host: 1.1.1.1", output
+      assert_match "Log entry", output
+    end
+  end
+
+  test "logs with follow" do
+    SSHKit::Backend::Abstract.any_instance.stubs(:exec)
+      .with("ssh -t root@1.1.1.1 -p 22 'docker logs kamal-proxy --timestamps --tail 10 --follow 2>&1'")
+
+    assert_match "docker logs kamal-proxy --timestamps --tail 10 --follow", run_command("logs", "--follow")
+  end
+
+  test "remove" do
+    Kamal::Cli::Proxy.any_instance.expects(:stop)
+    Kamal::Cli::Proxy.any_instance.expects(:remove_container)
+    Kamal::Cli::Proxy.any_instance.expects(:remove_image)
+
+    run_command("remove")
+  end
+
+  test "remove_container" do
+    run_command("remove_container").tap do |output|
+      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy", output
+    end
+  end
+
+  test "remove_image" do
+    run_command("remove_image").tap do |output|
+      assert_match "docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy", output
+    end
+  end
+
+  test "commands disallowed when proxy is disabled" do
+    assert_raises_when_disabled "boot"
+    assert_raises_when_disabled "reboot"
+    assert_raises_when_disabled "start"
+    assert_raises_when_disabled "stop"
+    assert_raises_when_disabled "details"
+    assert_raises_when_disabled "logs"
+    assert_raises_when_disabled "remove"
+  end
+
+  private
+    def run_command(*command, fixture: :with_proxy)
+      stdouted { Kamal::Cli::Proxy.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) }
+    end
+
+    def assert_raises_when_disabled(command)
+      assert_raises "kamal proxy commands are disabled unless experimental proxy support is enabled. Use `kamal traefik` commands instead." do
+        run_command(command, fixture: :with_accessories)
+      end
+    end
+end

test/commands/proxy_test.rb

@@ -0,0 +1,126 @@
+require "test_helper"
+
+class CommandsProxyTest < ActiveSupport::TestCase
+  setup do
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ]
+    }
+
+    ENV["EXAMPLE_API_KEY"] = "456"
+  end
+
+  teardown do
+    ENV.delete("EXAMPLE_API_KEY")
+  end
+
+  test "run" do
+    assert_equal \
+      "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}",
+      new_command.run.join(" ")
+  end
+
+  test "run with ports configured" do
+    assert_equal \
+      "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}",
+      new_command.run.join(" ")
+  end
+
+  test "run without configuration" do
+    @config.delete(:proxy)
+
+    assert_equal \
+      "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}",
+      new_command.run.join(" ")
+  end
+
+  test "run with logging config" do
+    @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
+
+    assert_equal \
+      "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}",
+      new_command.run.join(" ")
+  end
+
+  test "proxy start" do
+    assert_equal \
+      "docker container start kamal-proxy",
+      new_command.start.join(" ")
+  end
+
+  test "proxy stop" do
+    assert_equal \
+      "docker container stop kamal-proxy",
+      new_command.stop.join(" ")
+  end
+
+  test "proxy info" do
+    assert_equal \
+      "docker ps --filter name=^kamal-proxy$",
+      new_command.info.join(" ")
+  end
+
+  test "proxy logs" do
+    assert_equal \
+      "docker logs kamal-proxy --timestamps 2>&1",
+      new_command.logs.join(" ")
+  end
+
+  test "proxy logs since 2h" do
+    assert_equal \
+      "docker logs kamal-proxy  --since 2h --timestamps 2>&1",
+      new_command.logs(since: "2h").join(" ")
+  end
+
+  test "proxy logs last 10 lines" do
+    assert_equal \
+      "docker logs kamal-proxy  --tail 10 --timestamps 2>&1",
+      new_command.logs(lines: 10).join(" ")
+  end
+
+  test "proxy logs with grep hello!" do
+    assert_equal \
+      "docker logs kamal-proxy --timestamps 2>&1 | grep 'hello!'",
+      new_command.logs(grep: "hello!").join(" ")
+  end
+
+  test "proxy remove container" do
+    assert_equal \
+      "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy",
+      new_command.remove_container.join(" ")
+  end
+
+  test "proxy remove image" do
+    assert_equal \
+      "docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy",
+      new_command.remove_image.join(" ")
+  end
+
+  test "proxy follow logs" do
+    assert_equal \
+      "ssh -t root@1.1.1.1 -p 22 'docker logs kamal-proxy --timestamps --tail 10 --follow 2>&1'",
+      new_command.follow_logs(host: @config[:servers].first)
+  end
+
+  test "proxy follow logs with grep hello!" do
+    assert_equal \
+      "ssh -t root@1.1.1.1 -p 22 'docker logs kamal-proxy --timestamps --tail 10 --follow 2>&1 | grep \"hello!\"'",
+      new_command.follow_logs(host: @config[:servers].first, grep: "hello!")
+  end
+
+  test "deploy" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy deploy service --target \"172.1.0.2:80\"",
+      new_command.deploy("service", target: "172.1.0.2:80").join(" ")
+  end
+
+  test "remove" do
+    assert_equal \
+      "docker exec kamal-proxy kamal-proxy remove service --target \"172.1.0.2:80\"",
+      new_command.remove("service", target: "172.1.0.2:80").join(" ")
+  end
+
+  private
+    def new_command
+      Kamal::Commands::Proxy.new(Kamal::Configuration.new(@config, version: "123"))
+    end
+end

test/fixtures/deploy_with_proxy.yml

@@ -0,0 +1,42 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+registry:
+  username: user
+  password: pw
+
+proxy:
+  enabled: true
+  hosts:
+    - "1.1.1.1"
+  deploy_timeout: 6s
+
+accessories:
+  mysql:
+    image: mysql:5.7
+    host: 1.1.1.3
+    port: 3306
+    env:
+      clear:
+        MYSQL_ROOT_HOST: '%'
+      secret:
+        - MYSQL_ROOT_PASSWORD
+    files:
+      - test/fixtures/files/my.cnf:/etc/mysql/my.cnf
+    directories:
+      - data:/var/lib/mysql
+  redis:
+    image: redis:latest
+    roles:
+      - web
+    port: 6379
+    directories:
+      - data:/data
+
+readiness_delay: 0

test/integration/broken_deploy_test.rb

@@ -29,6 +29,5 @@ class BrokenDeployTest < IntegrationTest
       assert_match /First web container is unhealthy on vm[12], not booting any other roles/, output
       assert_match "First web container is unhealthy, not booting workers on vm3", output
       assert_match "nginx: [emerg] unexpected end of file, expecting \";\" or \"}\" in /etc/nginx/conf.d/default.conf:2", output
-      assert_match 'ERROR {"Status":"unhealthy","FailingStreak":0,"Log":[]}', output
     end
 end

test/integration/docker/deployer/app/Dockerfile

@@ -6,4 +6,4 @@ ARG COMMIT_SHA
 RUN echo $COMMIT_SHA > /usr/share/nginx/html/version
 RUN mkdir -p /usr/share/nginx/html/versions && echo "version" > /usr/share/nginx/html/versions/$COMMIT_SHA
 RUN mkdir -p /usr/share/nginx/html/versions && echo "hidden" > /usr/share/nginx/html/versions/.hidden
-
+RUN echo "Up!" > /usr/share/nginx/html/up

test/integration/docker/deployer/app_with_roles/Dockerfile

@@ -6,4 +6,4 @@ ARG COMMIT_SHA
 RUN echo $COMMIT_SHA > /usr/share/nginx/html/version
 RUN mkdir -p /usr/share/nginx/html/versions && echo "version" > /usr/share/nginx/html/versions/$COMMIT_SHA
 RUN mkdir -p /usr/share/nginx/html/versions && echo "hidden" > /usr/share/nginx/html/versions/.hidden
-
+RUN echo "Up!" > /usr/share/nginx/html/up

test/integration/docker/deployer/app_with_roles/config/deploy.yml

@@ -9,6 +9,11 @@ servers:
     hosts:
       - vm3
     cmd: sleep infinity
+proxy:
+  enabled: true
+  hosts:
+    - vm2
+  deploy_timeout: 2s
 
 asset_path: /usr/share/nginx/html/versions
 

test/integration/integration_test.rb

@@ -31,7 +31,7 @@ class IntegrationTest < ActiveSupport::TestCase
         succeeded = system("cd test/integration && #{command}")
       end
 
-      raise "Command `#{command}` failed with error code `#{$?}`" if !succeeded && raise_on_error
+      raise "Command `#{command}` failed with error code `#{$?}`, and output:\n#{result}" if !succeeded && raise_on_error
       result
     end
 

test/integration/proxy_test.rb

@@ -0,0 +1,84 @@
+require_relative "integration_test"
+
+class ProxyTest < IntegrationTest
+  setup do
+    @app = "app_with_roles"
+  end
+
+  test "boot, reboot, stop, start, restart, logs, remove" do
+    kamal :envify
+
+    kamal :proxy, :boot
+    assert_proxy_running
+
+    output = kamal :proxy, :reboot, "-y", "--verbose", capture: true
+    assert_proxy_running
+    assert_hooks_ran "pre-traefik-reboot", "post-traefik-reboot"
+    assert_match /Rebooting Traefik on vm1,vm2.../, output
+    assert_match /Rebooted Traefik on vm1,vm2/, output
+
+    output = kamal :proxy, :reboot, "--rolling", "-y", "--verbose", capture: true
+    assert_proxy_running
+    assert_hooks_ran "pre-traefik-reboot", "post-traefik-reboot"
+    assert_match /Rebooting Traefik on vm1.../, output
+    assert_match /Rebooted Traefik on vm1/, output
+    assert_match /Rebooting Traefik on vm2.../, output
+    assert_match /Rebooted Traefik on vm2/, output
+
+    kamal :proxy, :boot
+    assert_proxy_running
+    assert_traefik_running
+
+    # Check booting when booted doesn't raise an error
+    kamal :proxy, :stop
+    assert_proxy_not_running
+    assert_traefik_not_running
+
+    # Check booting when stopped works
+    kamal :proxy, :boot
+    assert_proxy_running
+    assert_traefik_running
+
+    kamal :proxy, :stop
+    assert_proxy_not_running
+    assert_traefik_not_running
+
+    kamal :proxy, :start
+    assert_proxy_running
+    assert_traefik_running
+
+    kamal :proxy, :restart
+    assert_proxy_running
+    assert_traefik_running
+
+    logs = kamal :proxy, :logs, capture: true
+    assert_match /Traefik version [\d.]+ built on/, logs
+
+    kamal :proxy, :remove
+    assert_proxy_not_running
+    assert_traefik_not_running
+
+    kamal :env, :delete
+  end
+
+  private
+    def assert_proxy_running
+      assert_match /basecamp\/kamal-proxy:latest   \"kamal-proxy run\"/, proxy_details
+    end
+
+    def assert_proxy_not_running
+      assert_no_match /basecamp\/kamal-proxy:latest   \"kamal-proxy run\"/, proxy_details
+    end
+
+    def assert_traefik_running
+      assert_match /traefik:v2.10   "\/entrypoint.sh/, proxy_details
+    end
+
+    def assert_traefik_not_running
+      assert_no_match /traefik:v2.10   "\/entrypoint.sh/, proxy_details
+    end
+
+    def proxy_details
+      kamal :proxy, :details, capture: true
+    end
+end