From 7c7785c1eb7bd4aa942bf11673e813402dec1b69 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Tue, 17 Sep 2024 18:55:52 +0100 Subject: [PATCH] Downgrade from Kamal 2 to 1.9 Add a downgrade command, so you can reverse the upgrade process and go back to Kamal 1.9. This replaces kamal-proxy and reboots all the accessories. This gives an upgrade and downgrade path: Upgrade: 1. Upgrade config to be Kamal 2 compatible + use kamal 2.0 2. Run `kamal upgrade` Downgrade: 1. Switch back to previous config + use kamal 1.9 2. Run `kamal downgrade` You can set `--rolling` to downgrade one host at a time. --- lib/kamal/cli/accessory.rb | 19 ++++++++ lib/kamal/cli/base.rb | 4 ++ lib/kamal/cli/main.rb | 31 ++++++++++++ lib/kamal/cli/traefik.rb | 40 +++++++++++++++ lib/kamal/commander.rb | 7 +++ lib/kamal/commander/specifics.rb | 2 +- lib/kamal/commands/traefik.rb | 9 ++++ test/cli/accessory_test.rb | 18 +++++++ test/cli/main_test.rb | 28 +++++++++++ test/cli/traefik_test.rb | 84 ++++++++++++++++++++++++++++++++ 10 files changed, 241 insertions(+), 1 deletion(-) diff --git a/lib/kamal/cli/accessory.rb b/lib/kamal/cli/accessory.rb index b3ff10f5..9cc2ca00 100644 --- a/lib/kamal/cli/accessory.rb +++ b/lib/kamal/cli/accessory.rb @@ -222,6 +222,25 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base end end + desc "downgrade", "Downgrade accessories from Kamal 2 to 1.9" + option :rolling, type: :boolean, default: false, desc: "Upgrade one host at a time" + option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question" + def downgrade(name) + confirming "This will restart all accessories" do + with_lock do + host_groups = options[:rolling] ? KAMAL.accessory_hosts : [ KAMAL.accessory_hosts ] + host_groups.each do |hosts| + host_list = Array(hosts).join(",") + KAMAL.with_specific_hosts(hosts) do + say "Downgrading #{name} accessories on #{host_list}...", :magenta + reboot name + say "Downgraded #{name} accessories on #{host_list}...", :magenta + end + end + end + end + end + private def with_accessory(name) if KAMAL.config.accessory(name) diff --git a/lib/kamal/cli/base.rb b/lib/kamal/cli/base.rb index 0710c088..ab67a0e9 100644 --- a/lib/kamal/cli/base.rb +++ b/lib/kamal/cli/base.rb @@ -206,6 +206,10 @@ module Kamal::Cli instance_variable_get("@_invocations").first end + def reset_invocation(cli_class) + instance_variable_get("@_invocations")[cli_class].pop + end + def ensure_run_and_locks_directory on(KAMAL.hosts) do execute(*KAMAL.server.ensure_run_directory) diff --git a/lib/kamal/cli/main.rb b/lib/kamal/cli/main.rb index 598d6c42..42281169 100644 --- a/lib/kamal/cli/main.rb +++ b/lib/kamal/cli/main.rb @@ -217,6 +217,37 @@ class Kamal::Cli::Main < Kamal::Cli::Base end end + desc "downgrade", "Downgrade from Kamal 2 to 1.9" + option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question" + option :rolling, type: :boolean, default: false, desc: "Downgrade one host at a time" + def downgrade + confirming "This will replace Traefik with kamal-proxy and restart all accessories" do + with_lock do + if options[:rolling] + (KAMAL.hosts | KAMAL.accessory_hosts).each do |host| + KAMAL.with_specific_hosts(host) do + say "Downgrading #{host}...", :magenta + if KAMAL.hosts.include?(host) + invoke "kamal:cli:traefik:downgrade", [], options.merge(confirmed: true, rolling: false) + reset_invocation(Kamal::Cli::Traefik) + end + if KAMAL.accessory_hosts.include?(host) + invoke "kamal:cli:accessory:downgrade", [ "all" ], options.merge(confirmed: true, rolling: false) + reset_invocation(Kamal::Cli::Accessory) + end + say "Downgraded #{host}", :magenta + end + end + else + say "Downgrading all hosts...", :magenta + invoke "kamal:cli:traefik:downgrade", [], options.merge(confirmed: true) + invoke "kamal:cli:accessory:downgrade", [ "all" ], options.merge(confirmed: true) + say "Downgraded all hosts", :magenta + end + end + end + end + desc "version", "Show Kamal version" def version puts Kamal::VERSION diff --git a/lib/kamal/cli/traefik.rb b/lib/kamal/cli/traefik.rb index a8bd2126..cee97ce0 100644 --- a/lib/kamal/cli/traefik.rb +++ b/lib/kamal/cli/traefik.rb @@ -119,4 +119,44 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base end end end + + desc "downgrade", "Downgrade to Traefik on servers (stop container, remove container, start new container, reboot app)" + option :rolling, type: :boolean, default: false, desc: "Reboot proxy on hosts in sequence, rather than in parallel" + option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question" + def downgrade + invoke_options = { "version" => KAMAL.config.latest_tag }.merge(options) + + confirming "This will cause a brief outage on each host. Are you sure?" do + host_groups = options[:rolling] ? KAMAL.hosts : [ KAMAL.hosts ] + host_groups.each do |hosts| + host_list = Array(hosts).join(",") + say "Downgrading to Traefik on #{host_list}...", :magenta + run_hook "pre-traefik-reboot", hosts: host_list + on(hosts) do |host| + execute *KAMAL.auditor.record("Rebooted Traefik"), verbosity: :debug + execute *KAMAL.registry.login + + "Stopping and removing kamal-proxy on #{host}, if running..." + execute *KAMAL.traefik.cleanup_kamal_proxy + + "Stopping and removing Traefik on #{host}, if running..." + execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false + execute *KAMAL.traefik.remove_container + execute *KAMAL.traefik.remove_image + end + + KAMAL.with_specific_hosts(hosts) do + invoke "kamal:cli:traefik:boot", [], invoke_options + reset_invocation(Kamal::Cli::Traefik) + invoke "kamal:cli:app:boot", [], invoke_options + reset_invocation(Kamal::Cli::App) + invoke "kamal:cli:prune:all", [], invoke_options + reset_invocation(Kamal::Cli::Prune) + end + + run_hook "post-traefik-reboot", hosts: host_list + say "Downgraded to Traefik on #{host_list}", :magenta + end + end + end end diff --git a/lib/kamal/commander.rb b/lib/kamal/commander.rb index c28fda82..f9873f00 100644 --- a/lib/kamal/commander.rb +++ b/lib/kamal/commander.rb @@ -56,6 +56,13 @@ class Kamal::Commander end end + def with_specific_hosts(hosts) + original_hosts, self.specific_hosts = specific_hosts, hosts + yield + ensure + self.specific_hosts = original_hosts + end + def accessory_names config.accessories&.collect(&:name) || [] end diff --git a/lib/kamal/commander/specifics.rb b/lib/kamal/commander/specifics.rb index 127bd40e..7c713e98 100644 --- a/lib/kamal/commander/specifics.rb +++ b/lib/kamal/commander/specifics.rb @@ -23,7 +23,7 @@ class Kamal::Commander::Specifics end def accessory_hosts - specific_hosts || config.accessories.flat_map(&:hosts) + config.accessories.flat_map(&:hosts) & specified_hosts end private diff --git a/lib/kamal/commands/traefik.rb b/lib/kamal/commands/traefik.rb index 07e0e6ea..270dc04b 100644 --- a/lib/kamal/commands/traefik.rb +++ b/lib/kamal/commands/traefik.rb @@ -62,6 +62,15 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base [ :rm, "-f", env.secrets_file ] end + def cleanup_kamal_proxy + chain \ + docker(:container, :stop, "kamal-proxy"), + combine( + docker(:container, :prune, "--force", "--filter", "label=org.opencontainers.image.title=kamal-proxy"), + docker(:image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=kamal-proxy") + ) + end + private def publish_args argumentize "--publish", port if publish? diff --git a/test/cli/accessory_test.rb b/test/cli/accessory_test.rb index e56eef2d..fcf9a2e6 100644 --- a/test/cli/accessory_test.rb +++ b/test/cli/accessory_test.rb @@ -209,6 +209,24 @@ class CliAccessoryTest < CliTestCase end end + test "downgrade" do + run_command("downgrade", "-y", "all").tap do |output| + assert_match "Downgrading all accessories on 1.1.1.3,1.1.1.1,1.1.1.2...", output + assert_match "docker container stop app-mysql on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "Downgraded all accessories on 1.1.1.3,1.1.1.1,1.1.1.2", output + end + end + + test "downgrade rolling" do + run_command("downgrade", "--rolling", "-y", "all").tap do |output| + assert_match "Downgrading all accessories on 1.1.1.3...", output + assert_match "docker container stop app-mysql on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "Downgraded all accessories on 1.1.1.3", output + end + end + private def run_command(*command) stdouted { Kamal::Cli::Accessory.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) } diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index e562499b..0e09074f 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -537,6 +537,34 @@ class CliMainTest < CliTestCase assert_equal Kamal::VERSION, version end + test "downgrade" do + invoke_options = { "config_file" => "test/fixtures/deploy_with_accessories.yml", "skip_hooks" => false, "confirmed" => true, "rolling" => false } + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:downgrade", [], invoke_options) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:downgrade", [ "all" ], invoke_options) + + run_command("downgrade", "-y", config_file: "deploy_with_accessories").tap do |output| + assert_match "Downgrading all hosts...", output + assert_match "Downgraded all hosts", output + end + end + + test "downgrade rolling" do + invoke_options = { "config_file" => "test/fixtures/deploy_with_accessories.yml", "skip_hooks" => false, "confirmed" => true, "rolling" => false } + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:downgrade", [], invoke_options).times(4) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:downgrade", [ "all" ], invoke_options).times(3) + + run_command("downgrade", "--rolling", "-y", config_file: "deploy_with_accessories").tap do |output| + assert_match "Downgrading 1.1.1.1...", output + assert_match "Downgraded 1.1.1.1", output + assert_match "Downgrading 1.1.1.2...", output + assert_match "Downgraded 1.1.1.2", output + assert_match "Downgrading 1.1.1.3...", output + assert_match "Downgraded 1.1.1.3", output + assert_match "Downgrading 1.1.1.4...", output + assert_match "Downgraded 1.1.1.4", output + end + end + private def run_command(*command, config_file: "deploy_simple") stdouted { Kamal::Cli::Main.start([ *command, "-c", "test/fixtures/#{config_file}.yml" ]) } diff --git a/test/cli/traefik_test.rb b/test/cli/traefik_test.rb index 29171150..c01c9bb5 100644 --- a/test/cli/traefik_test.rb +++ b/test/cli/traefik_test.rb @@ -103,6 +103,90 @@ class CliTraefikTest < CliTestCase end end + test "downgrade" do + Object.any_instance.stubs(:sleep) + + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false) + .returns("12345678") + + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", raise_on_non_zero_exit: false) + .returns("12345678") + + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) + .with { |*args| args[0..1] == [ :sh, "-c" ] } + .returns("123") # old version + + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("running") # health check + + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("running").at_least_once # workers health check + + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) + .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false) + .returns("") # old version + + run_command("downgrade", "-y").tap do |output| + assert_match "Downgrading to Traefik on 1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4...", output + assert_match "docker login -u [REDACTED] -p [REDACTED]", output + assert_match "docker container stop kamal-proxy ; docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy && docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy", output + assert_match "docker container stop traefik", output + assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik", output + assert_match "docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik", output + assert_match "/usr/bin/env mkdir -p .kamal", output + assert_match "docker login -u [REDACTED] -p [REDACTED]", output + assert_match "docker container start traefik || docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" traefik:v2.10 --providers.docker --log.level=\"DEBUG\"", output + assert_match "/usr/bin/env mkdir -p .kamal", output + assert_match %r{docker rename app-web-latest app-web-latest_replaced_.*}, output + assert_match %r{docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-.* -e KAMAL_CONTAINER_NAME="app-web-latest" -e KAMAL_VERSION="latest" --env-file .kamal/env/roles/app-web.env --health-cmd}, output + assert_match "docker tag dhh/app:latest dhh/app:latest", output + assert_match "/usr/bin/env mkdir -p .kamal", output + assert_match "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done", output + assert_match "docker image prune --force --filter label=service=app", output + assert_match "Downgraded to Traefik on 1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", output + end + end + + test "downgrade rolling" do + Object.any_instance.stubs(:sleep) + + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false) + .returns("12345678") + + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", raise_on_non_zero_exit: false) + .returns("12345678") + + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) + .with { |*args| args[0..1] == [ :sh, "-c" ] } + .returns("123") # old version + + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("running") # health check + + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("running").at_least_once # workers health check + + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) + .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false) + .returns("") # old version + + run_command("downgrade", "--rolling", "-y",).tap do |output| + %w[1.1.1.1 1.1.1.2 1.1.1.3 1.1.1.4].each do |host| + assert_match "Downgrading to Traefik on #{host}...", output + assert_match "docker container stop kamal-proxy ; docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy && docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy", output + assert_match "Downgraded to Traefik on #{host}", output + end + end + end + private def run_command(*command) stdouted { Kamal::Cli::Traefik.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }