Upgrade commands for Kamal 1.x -> 2.0

Adds:
- `kamal upgrade` to upgrade all app hosts and accessory hosts
- `kamal proxy upgrade` to upgrade the proxy on all hosts
- `kamal accessory upgrade [name]` to upgrade accessories on all hosts

Upgrade takes rolling and confirmed options and calls `proxy upgrade`
and `accessory upgrade` in turn.

To just upgrade a single host add -h [host] to the command. But the
upgrade should run on all hosts, not just those running the proxy.

Calling upgrade on a host that has already been upgraded should work ok.

Upgrading hosts causes downtime but you can avoid if you run multiple
hosts by:
1. Implementing the pre-proxy-reboot and post-proxy-reboot hooks to
   remove the host from external load balancers
2. Running the upgrade with the --rolling option

**kamal proxy upgrade**
1. Creates a `kamal` network if required
2. Stops and removes the old proxy (whether Traefik or kamal-proxy)
3. Starts a kamal-proxy container in the `kamal` network
4. Reboots the app containers in the `kamal` network

**kamal accessory upgrade [name]**
1. Creates a `kamal` network if required
2. Reboots the accessory containers in the `kamal` network

A matching `downgrade` command will be added to Kamal 1.9.

lib/kamal/cli/accessory.rb

@@ -218,6 +218,29 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     end
   end
 
+  desc "upgrade", "Upgrade accessories from Kamal 1.x to 2.0 (restart them in 'kamal' network)"
+  option :rolling, type: :boolean, default: false, desc: "Upgrade one host at a time"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  def upgrade(name)
+    confirming "This will restart all accessories" do
+      with_lock do
+        if options[:rolling]
+          KAMAL.accessory_hosts.each do |host|
+            say "Upgrading accessories on #{host}...", :magenta
+            KAMAL.with_specific_hosts(host) do
+              reboot name
+            end
+            say "Upgraded accessories on #{host}...", :magenta
+          end
+        else
+          say "Upgrading accessories on all hosts...", :magenta
+          reboot name
+          say "Upgraded accessories on all hosts", :magenta
+        end
+      end
+    end
+  end
+
   private
     def with_accessory(name)
       if KAMAL.config.accessory(name)

lib/kamal/cli/base.rb

@@ -174,6 +174,10 @@ module Kamal::Cli
         instance_variable_get("@_invocations").first
       end
 
+      def reset_invocation(cli_class)
+        instance_variable_get("@_invocations")[cli_class].pop
+      end
+
       def ensure_run_directory
         on(KAMAL.hosts) do
           execute(*KAMAL.server.ensure_run_directory)

lib/kamal/cli/main.rb

@@ -189,6 +189,37 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     end
   end
 
+  desc "upgrade", "Upgrade from Kamal 1.x to 2.0"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  option :rolling, type: :boolean, default: false, desc: "Upgrade one host at a time"
+  def upgrade
+    confirming "This will replace Traefik with kamal-proxy and restart all accessories" do
+      with_lock do
+        if options[:rolling]
+          (KAMAL.hosts | KAMAL.accessory_hosts).each do |host|
+            KAMAL.with_specific_hosts(host) do
+              say "Upgrading #{host}...", :magenta
+              if KAMAL.hosts.include?(host)
+                invoke "kamal:cli:proxy:upgrade", [], options.merge(confirmed: true, rolling: false)
+                reset_invocation(Kamal::Cli::Proxy)
+              end
+              if KAMAL.accessory_hosts.include?(host)
+                invoke "kamal:cli:accessory:upgrade", [ "all" ], options.merge(confirmed: true, rolling: false)
+                reset_invocation(Kamal::Cli::Accessory)
+              end
+              say "Upgraded #{host}", :magenta
+            end
+          end
+        else
+          say "Upgrading all hosts...", :magenta
+          invoke "kamal:cli:proxy:upgrade", [], options.merge(confirmed: true)
+          invoke "kamal:cli:accessory:upgrade", [ "all" ], options.merge(confirmed: true)
+          say "Upgraded all hosts", :magenta
+        end
+      end
+    end
+  end
+
   desc "version", "Show Kamal version"
   def version
     puts Kamal::VERSION

lib/kamal/cli/proxy.rb

@@ -62,7 +62,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
     end
   end
 
-  desc "upgrade", "Upgrade to correct proxy on servers (stop container, remove container, start new container)"
+  desc "upgrade", "Upgrade to kamal-proxy on servers (stop container, remove container, start new container, reboot app)"
   option :rolling, type: :boolean, default: false, desc: "Reboot proxy on hosts in sequence, rather than in parallel"
   option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def upgrade
@@ -72,6 +72,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
       host_groups = options[:rolling] ? KAMAL.hosts : [ KAMAL.hosts ]
       host_groups.each do |hosts|
         host_list = Array(hosts).join(",")
+        say "Upgrading proxy on #{host_list}..."
         run_hook "pre-proxy-reboot", hosts: host_list
         on(hosts) do |host|
           execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
@@ -86,19 +87,17 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
           execute *KAMAL.proxy.remove_image
         end
 
-        begin
+        KAMAL.with_specific_hosts(hosts) do
-          old_hosts, KAMAL.specific_hosts = KAMAL.specific_hosts, hosts
           invoke "kamal:cli:proxy:boot", [], invoke_options
           reset_invocation(Kamal::Cli::Proxy)
           invoke "kamal:cli:app:boot", [], invoke_options
           reset_invocation(Kamal::Cli::App)
           invoke "kamal:cli:prune:all", [], invoke_options
           reset_invocation(Kamal::Cli::Prune)
-        ensure
-          KAMAL.specific_hosts = old_hosts
         end
 
         run_hook "post-proxy-reboot", hosts: host_list
+        say "Upgraded proxy on #{host_list}"
       end
     end
   end
@@ -204,10 +203,6 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
   end
 
   private
-    def reset_invocation(cli_class)
-      instance_variable_get("@_invocations")[cli_class].pop
-    end
-
     def removal_allowed?(force)
       on(KAMAL.proxy_hosts) do |host|
         app_count = capture_with_info(*KAMAL.server.app_directory_count).chomp.to_i

lib/kamal/commander.rb

@@ -65,6 +65,13 @@ class Kamal::Commander
     end
   end
 
+  def with_specific_hosts(hosts)
+    original_hosts, self.specific_hosts = specific_hosts, hosts
+    yield
+  ensure
+    self.specific_hosts = original_hosts
+  end
+
   def accessory_names
     config.accessories&.collect(&:name) || []
   end

test/cli/accessory_test.rb

@@ -220,6 +220,27 @@ class CliAccessoryTest < CliTestCase
     end
   end
 
+  test "upgrade" do
+    run_command("upgrade", "-y", "all").tap do |output|
+      assert_match "Upgrading accessories on all hosts...", output
+      assert_match "docker network create kamal on 1.1.1.3", output
+      assert_match "docker container stop app-mysql on 1.1.1.3", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST="%" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "Upgraded accessories on all hosts", output
+    end
+  end
+
+  test "upgrade rolling" do
+    run_command("upgrade", "--rolling", "-y", "all").tap do |output|
+      assert_match "Upgrading accessories on 1.1.1.3...", output
+      assert_match "docker network create kamal on 1.1.1.3", output
+      assert_match "docker container stop app-mysql on 1.1.1.3", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST="%" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "Upgraded accessories on 1.1.1.3", output
+    end
+  end
+
+
   private
     def run_command(*command)
       stdouted { Kamal::Cli::Accessory.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }

test/cli/main_test.rb

@@ -486,6 +486,34 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "upgrade" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_with_accessories.yml", "skip_hooks" => false, "confirmed" => true, "rolling" => false }
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:upgrade", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:upgrade", [ "all" ], invoke_options)
+
+    run_command("upgrade", "-y", config_file: "deploy_with_accessories").tap do |output|
+      assert_match "Upgrading all hosts...", output
+      assert_match "Upgraded all hosts", output
+    end
+  end
+
+  test "upgrade rolling" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_with_accessories.yml", "skip_hooks" => false, "confirmed" => true, "rolling" => false }
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:upgrade", [], invoke_options).times(4)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:upgrade", [ "all" ], invoke_options).times(4)
+
+    run_command("upgrade", "--rolling", "-y", config_file: "deploy_with_accessories").tap do |output|
+      assert_match "Upgrading 1.1.1.1...", output
+      assert_match "Upgraded 1.1.1.1", output
+      assert_match "Upgrading 1.1.1.2...", output
+      assert_match "Upgraded 1.1.1.2", output
+      assert_match "Upgrading 1.1.1.3...", output
+      assert_match "Upgraded 1.1.1.3", output
+      assert_match "Upgrading 1.1.1.4...", output
+      assert_match "Upgraded 1.1.1.4", output
+    end
+  end
+
   private
     def run_command(*command, config_file: "deploy_simple")
       with_argv([ *command, "-c", "test/fixtures/#{config_file}.yml" ]) do

test/cli/proxy_test.rb

@@ -182,6 +182,67 @@ class CliProxyTest < CliTestCase
     end
   end
 
+  test "upgrade" do
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("12345678")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2")
+      .returns("v0.1.0")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running").at_least_once # workers health check
+
+    run_command("upgrade", "-y").tap do |output|
+      assert_match "Upgrading proxy on 1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4...", output
+      assert_match "docker login -u [REDACTED] -p [REDACTED]", output
+      assert_match "docker container stop traefik ; docker container prune --force --filter label=org.opencontainers.image.title=Traefik && docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik", output
+      assert_match "docker container stop kamal-proxy", output
+      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy", output
+      assert_match "docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy", output
+      assert_match "/usr/bin/env mkdir -p .kamal", output
+      assert_match "docker network create kamal", output
+      assert_match "docker login -u [REDACTED] -p [REDACTED]", output
+      assert_match "docker container start kamal-proxy || docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume $(pwd)/.kamal/proxy/config:/home/kamal-proxy/.config/kamal-proxy --log-opt max-size=\"10m\" basecamp/kamal-proxy:v0.1.0", output
+      assert_match "/usr/bin/env mkdir -p .kamal", output
+      assert_match %r{docker rename app-web-latest app-web-latest_replaced_.*}, output
+      assert_match "/usr/bin/env mkdir -p .kamal/apps/app/env/roles", output
+      assert_match %r{/usr/bin/env .* .kamal/apps/app/env/roles/web.env}, output
+      assert_match %r{docker run --detach --restart unless-stopped --name app-web-latest --network kamal --hostname 1.1.1.1-.* -e KAMAL_CONTAINER_NAME="app-web-latest" -e KAMAL_VERSION="latest" --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size="10m" --label service="app" --label role="web" --label destination dhh/app:latest}, output
+      assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target \"12345678:80\" --deploy-timeout \"6s\" --buffer-requests --buffer-responses --log-request-header \"Cache-Control\" --log-request-header \"Last-Modified\" --log-request-header \"User-Agent\"", output
+      assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output
+      assert_match "docker tag dhh/app:latest dhh/app:latest", output
+      assert_match "/usr/bin/env mkdir -p .kamal", output
+      assert_match "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done", output
+      assert_match "docker image prune --force --filter label=service=app", output
+      assert_match "Upgraded proxy on 1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", output
+    end
+  end
+
+  test "upgrade rolling" do
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("12345678")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "kamal-proxy", "--format '{{.Config.Image}}'", "|", :cut, "-d:", "-f2")
+      .returns("v0.1.0")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running").at_least_once # workers health check
+
+    run_command("upgrade", "--rolling", "-y",).tap do |output|
+      %w[1.1.1.1 1.1.1.2 1.1.1.3 1.1.1.4].each do |host|
+        assert_match "Upgrading proxy on #{host}...", output
+        assert_match "docker container stop traefik ; docker container prune --force --filter label=org.opencontainers.image.title=Traefik && docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik on #{host}", output
+        assert_match "Upgraded proxy on #{host}", output
+      end
+    end
+  end
+
   private
     def run_command(*command, fixture: :with_proxy)
       stdouted { Kamal::Cli::Proxy.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) }

test/fixtures/deploy_with_proxy.yml

@@ -39,3 +39,4 @@ accessories:
       - data:/data
 
 readiness_delay: 0
+readiness_timeout: 1