From 579b4cd9aa351b3f96818c219535793ff50a2b26 Mon Sep 17 00:00:00 2001 From: Jacopo Date: Tue, 11 Apr 2023 10:22:03 +0200 Subject: [PATCH] Simplify By using and ad-hoc command to detect and stop stale containers. By default stale containers are only detected. --- lib/mrsk/cli/app.rb | 67 ++++++++++++++++++++++++---------------- lib/mrsk/cli/main.rb | 6 ++++ lib/mrsk/commands/app.rb | 3 +- test/cli/app_test.rb | 46 +++++++++++++-------------- test/cli/main_test.rb | 15 +++++---- 5 files changed, 76 insertions(+), 61 deletions(-) diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb index 2ac92213..b8aa7195 100644 --- a/lib/mrsk/cli/app.rb +++ b/lib/mrsk/cli/app.rb @@ -8,13 +8,13 @@ class Mrsk::Cli::App < Mrsk::Cli::Base cli = self - MRSK.hosts.each do |host| + on(MRSK.hosts) do |host| roles = MRSK.roles_on(host) roles.each do |role| - on(host) do - execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug + execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug + begin if capture_with_info(*MRSK.app(role: role).container_id_for_version(version)).present? tmp_version = "#{version}_#{SecureRandom.hex(8)}" info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}" @@ -22,16 +22,10 @@ class Mrsk::Cli::App < Mrsk::Cli::Base execute *MRSK.app(role: role).rename_container(version: version, new_version: tmp_version) end + old_version = capture_with_info(*MRSK.app(role: role).current_running_version, raise_on_non_zero_exit: false).strip execute *MRSK.app(role: role).run sleep MRSK.config.readiness_delay - end - - old_versions = list_versions(host: host, role: role, only_old: true) - old_versions.each do |old_version| - on(host) do - info "Stopping old container with version #{old_version}" - execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false - end + execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present? end end end @@ -54,20 +48,14 @@ class Mrsk::Cli::App < Mrsk::Cli::Base end desc "stop", "Stop app container on servers" - option :only_old, aliases: "-o", type: :boolean, default: false, desc: "Stop only old versions containers" def stop with_lock do - MRSK.hosts.each do |host| + on(MRSK.hosts) do |host| roles = MRSK.roles_on(host) roles.each do |role| - audit_record = options[:only_old] ? "Stopped old containers" : "Stopped app" - on(host) { execute *MRSK.auditor(role: role).record(audit_record), verbosity: :debug } - - versions = list_versions(host: host, role: role, only_old: options[:only_old]) - versions.each do |version| - on(host) { execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false } - end + execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug + execute *MRSK.app(role: role).stop, raise_on_non_zero_exit: false end end end @@ -136,6 +124,35 @@ class Mrsk::Cli::App < Mrsk::Cli::Base on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_containers) } end + desc "stale_containers", "Detect app stale containers" + option :stop, aliases: "-s", type: :boolean, default: false, desc: "Stop the stale containers found." + def stale_containers + with_lock do + stop = options[:stop] + + on(MRSK.hosts) do |host| + roles = MRSK.roles_on(host) + + roles.each do |role| + stale_versions = \ + capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false) + .split("\n") + .map(&:strip) + .drop(1) + + stale_versions.each do |version| + if stop + puts_by_host host, "Stopping stale container with version #{version}" + execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false + else + puts_by_host host, "Detected stale container with version #{version} (use `mrsk app stale_containers --stop` to stop)" + end + end + end + end + end + end + desc "images", "Show app images on servers" def images on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_images) } @@ -247,13 +264,9 @@ class Mrsk::Cli::App < Mrsk::Cli::Base end def current_running_version(host: MRSK.primary_host) - list_versions(host: host, status: :running).shift.presence - end - - def list_versions(host:, role: nil, status: nil, only_old: false) - versions = nil - on(host) { versions = capture_with_info(*MRSK.app(role: role).list_versions(status: status), raise_on_non_zero_exit: false).split("\n").map(&:strip) } - only_old ? versions.drop(1) : versions + version = nil + on(host) { version = capture_with_info(*MRSK.app.current_running_version).strip } + version.presence end def version_or_latest diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index 83f71c34..e39f923c 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -37,6 +37,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base say "Ensure app can pass healthcheck...", :magenta invoke "mrsk:cli:healthcheck:perform", [], invoke_options + say "Detect stale containers...", :magenta + invoke "mrsk:cli:app:stale_containers", [], invoke_options + invoke "mrsk:cli:app:boot", [], invoke_options say "Prune old containers and images...", :magenta @@ -65,6 +68,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base say "Ensure app can pass healthcheck...", :magenta invoke "mrsk:cli:healthcheck:perform", [], invoke_options + say "Detect stale containers...", :magenta + invoke "mrsk:cli:app:stale_containers", [], invoke_options + invoke "mrsk:cli:app:boot", [], invoke_options end diff --git a/lib/mrsk/commands/app.rb b/lib/mrsk/commands/app.rb index fc192038..7f435934 100644 --- a/lib/mrsk/commands/app.rb +++ b/lib/mrsk/commands/app.rb @@ -97,8 +97,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base def list_versions(*docker_args, status: nil) pipe \ docker(:ps, *filter_args(status: status), *docker_args, "--format", '"{{.Names}}"'), - # Extract SHA from "service-role-dest-SHA" - %(grep -oE "\\-[^-]+$"), + %(grep -oE "\\-[^-]+$"), # Extract SHA from "service-role-dest-SHA" %(cut -c 2-) end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 245b7246..53925717 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -2,13 +2,8 @@ require_relative "cli_test_case" class CliAppTest < CliTestCase test "boot" do - # current version not running yet - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet") - - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false) - .returns("124\n123") # running + old version + # Stub current version fetch + SSHKit::Backend::Abstract.any_instance.stubs(:capture).returns("123") # old version run_command("boot").tap do |output| assert_match "docker run --detach --restart unless-stopped", output @@ -21,17 +16,17 @@ class CliAppTest < CliTestCase SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet") - .returns("12345678") # current version already running + .returns("12345678") # running version SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false) - .returns("123\n12345678_1") # running + renamed version + .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false) + .returns("123") # old version run_command("boot").tap do |output| assert_match /Renaming container .* to .* as already deployed on 1.1.1.1/, output # Rename assert_match /docker rename .* .*/, output assert_match "docker run --detach --restart unless-stopped", output - assert_match "docker container ls --all --filter name=^app-web-12345678_1$ --quiet | xargs docker stop", output + assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output end ensure Thread.report_on_exception = true @@ -44,21 +39,28 @@ class CliAppTest < CliTestCase end test "stop" do - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false) - .returns("12345678") - run_command("stop").tap do |output| - assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output + assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop", output end end - test "stop only old containers" do + test "stale_containers" do SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false) .returns("12345678\n87654321") - run_command("stop", '--only-old').tap do |output| + run_command("stale_containers").tap do |output| + assert_match /Detected stale container with version 87654321/, output + end + end + + test "stop stale_containers" do + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false) + .returns("12345678\n87654321") + + run_command("stale_containers", "--stop").tap do |output| + assert_match /Stopping stale container with version 87654321/, output assert_match /#{Regexp.escape("docker container ls --all --filter name=^app-web-87654321$ --quiet | xargs docker stop")}/, output end end @@ -70,12 +72,8 @@ class CliAppTest < CliTestCase end test "remove" do - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false) - .returns("12345678") - run_command("remove").tap do |output| - assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output + assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop")}/, output assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output end @@ -107,7 +105,7 @@ class CliAppTest < CliTestCase test "exec with reuse" do run_command("exec", "--reuse", "ruby -v").tap do |output| - assert_match "docker ps --filter label=service=app --filter status=running --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output # Get current version + assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output # Get current version assert_match "docker exec app-web-999 ruby -v", output end end diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index 112e9f67..289fe866 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -17,6 +17,7 @@ class CliMainTest < CliTestCase Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options) + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options) @@ -26,6 +27,7 @@ class CliMainTest < CliTestCase assert_match /Build and push app image/, output assert_match /Ensure Traefik is running/, output assert_match /Ensure app can pass healthcheck/, output + assert_match /Detect stale containers/, output assert_match /Prune old containers and images/, output end end @@ -38,6 +40,7 @@ class CliMainTest < CliTestCase Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options) + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options) @@ -48,6 +51,7 @@ class CliMainTest < CliTestCase assert_match /Pull app image/, output assert_match /Ensure Traefik is running/, output assert_match /Ensure app can pass healthcheck/, output + assert_match /Detect stale containers/, output assert_match /Prune old containers and images/, output assert_match /Releasing the deploy lock/, output end @@ -58,6 +62,7 @@ class CliMainTest < CliTestCase Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options) + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options) run_command("redeploy").tap do |output| @@ -71,6 +76,7 @@ class CliMainTest < CliTestCase Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options) + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options) run_command("redeploy", "--skip_push").tap do |output| @@ -224,19 +230,12 @@ class CliMainTest < CliTestCase end test "remove with confirmation" do - %w[ web workers ].each do |role| - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false) - .times(2) - .returns("12345678") - end - run_command("remove", "-y", config_file: "deploy_with_accessories").tap do |output| assert_match /docker container stop traefik/, output assert_match /docker container prune --force --filter label=org.opencontainers.image.title=Traefik/, output assert_match /docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik/, output - assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output + assert_match /docker ps --quiet --filter label=service=app | xargs docker stop/, output assert_match /docker container prune --force --filter label=service=app/, output assert_match /docker image prune --all --force --filter label=service=app/, output