From e2f6db5caec094cc329d2f2c7b28cd38292157b1 Mon Sep 17 00:00:00 2001 From: Jacopo Date: Wed, 5 Apr 2023 19:03:27 +0200 Subject: [PATCH] Clear stale containers By stopping all the older containers with matching /#{service}-#{role}-#{dest}-.*/ running on the same host. --- lib/mrsk/cli/app.rb | 36 ++++++++++++++++++++++++++------ lib/mrsk/cli/main.rb | 6 ++++++ lib/mrsk/commands/app.rb | 27 +++++++++++++----------- test/cli/app_test.rb | 39 +++++++++++++++++++++++++--------- test/cli/main_test.rb | 19 ++++++++++++++--- test/commands/app_test.rb | 44 ++++++++++++++++++++++++--------------- 6 files changed, 123 insertions(+), 48 deletions(-) diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb index fe2599a1..a9dfb75a 100644 --- a/lib/mrsk/cli/app.rb +++ b/lib/mrsk/cli/app.rb @@ -50,12 +50,32 @@ class Mrsk::Cli::App < Mrsk::Cli::Base desc "stop", "Stop app container on servers" def stop with_lock do - on(MRSK.hosts) do |host| + MRSK.hosts.each do |host| roles = MRSK.roles_on(host) roles.each do |role| - execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug - execute *MRSK.app(role: role).stop, raise_on_non_zero_exit: false + on(host) { execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug } + all_versions = list_versions(host: host, role: role) + all_versions.each do |version| + on(host) { execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false } + end + end + end + end + end + + desc "stop_stale_containers", "Stop all app stale containers" + def stop_stale_containers + with_lock do + MRSK.hosts.each do |host| + roles = MRSK.roles_on(host) + + roles.each do |role| + stale_versions = list_versions(host: host, role: role).drop(1) + stale_versions.each do |version| + say "Stopping stale container with version #{version}", :red + on(host) { execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false } + end end end end @@ -235,9 +255,13 @@ class Mrsk::Cli::App < Mrsk::Cli::Base end def current_running_version(host: MRSK.primary_host) - version = nil - on(host) { version = capture_with_info(*MRSK.app.current_running_version).strip } - version.presence + list_versions(host: host, status: :running).shift.presence + end + + def list_versions(host:, role: nil, status: nil) + versions = nil + on(host) { versions = capture_with_info(*MRSK.app(role: role).list_versions(status: status)).split("\n").map(&:strip) } + versions end def version_or_latest diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index 83f71c34..ac6a54d6 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -37,6 +37,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base say "Ensure app can pass healthcheck...", :magenta invoke "mrsk:cli:healthcheck:perform", [], invoke_options + say "Stop stale containers...", :magenta + invoke "mrsk:cli:app:stop_stale_containers", [], invoke_options + invoke "mrsk:cli:app:boot", [], invoke_options say "Prune old containers and images...", :magenta @@ -65,6 +68,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base say "Ensure app can pass healthcheck...", :magenta invoke "mrsk:cli:healthcheck:perform", [], invoke_options + say "Stop stale containers...", :magenta + invoke "mrsk:cli:app:stop_stale_containers", [], invoke_options + invoke "mrsk:cli:app:boot", [], invoke_options end diff --git a/lib/mrsk/commands/app.rb b/lib/mrsk/commands/app.rb index 224e7090..2f5a5657 100644 --- a/lib/mrsk/commands/app.rb +++ b/lib/mrsk/commands/app.rb @@ -29,7 +29,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base def stop(version: nil) pipe \ - version ? container_id_for_version(version) : current_container_id, + version ? container_id_for_version(version) : current_running_container_id, xargs(config.stop_wait_time ? docker(:stop, "-t", config.stop_wait_time) : docker(:stop)) end @@ -40,7 +40,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base def logs(since: nil, lines: nil, grep: nil) pipe \ - current_container_id, + current_running_container_id, "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1", ("grep '#{grep}'" if grep) end @@ -48,7 +48,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base def follow_logs(host:, grep: nil) run_over_ssh \ pipe( - current_container_id, + current_running_container_id, "xargs docker logs --timestamps --tail 10 --follow 2>&1", (%(grep "#{grep}") if grep) ), @@ -82,8 +82,8 @@ class Mrsk::Commands::App < Mrsk::Commands::Base end - def current_container_id - docker :ps, "--quiet", *filter_args + def current_running_container_id + docker :ps, "--quiet", *filter_args(status: :running), "--latest" end def container_id_for_version(version) @@ -91,11 +91,13 @@ class Mrsk::Commands::App < Mrsk::Commands::Base end def current_running_version - # FIXME: Find more graceful way to extract the version from "app-version" than using sed and tail! + list_versions("--latest", status: :running) + end + + def list_versions(*docker_args, status: nil) pipe \ - docker(:ps, *filter_args, "--format", '"{{.Names}}"'), - %(sed 's/-/\\n/g'), - "tail -n 1" + docker(:ps, *filter_args(status: status), *docker_args, "--format", '"{{.Names}}"'), + %(grep -oP "(?<=\-)[^-]+$") # Extract SHA from "service-role-dest-SHA" end def list_containers @@ -134,14 +136,15 @@ class Mrsk::Commands::App < Mrsk::Commands::Base [ config.service, role, config.destination, version || config.version ].compact.join("-") end - def filter_args - argumentize "--filter", filters + def filter_args(status: nil) + argumentize "--filter", filters(status: status) end - def filters + def filters(status: nil) [ "label=service=#{config.service}" ].tap do |filters| filters << "label=destination=#{config.destination}" if config.destination filters << "label=role=#{role}" if role + filters << "status=#{status}" if status end end end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 2dc5ad64..cb3ca638 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -19,7 +19,7 @@ class CliAppTest < CliTestCase .returns("12345678") # running version SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1") + .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"") .returns("123") # old version run_command("boot").tap do |output| @@ -39,8 +39,23 @@ class CliAppTest < CliTestCase end test "stop" do + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"") + .returns("12345678") + run_command("stop").tap do |output| - assert_match "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop", output + assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output + end + end + + test "stop_stale_containers" do + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"") + .returns("12345678\n87654321") + + run_command("stop_stale_containers").tap do |output| + assert_match /Stopping stale container with version 87654321/, output + assert_match /#{Regexp.escape("docker container ls --all --filter name=^app-web-87654321$ --quiet | xargs docker stop")}/, output end end @@ -51,8 +66,12 @@ class CliAppTest < CliTestCase end test "remove" do + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"") + .returns("12345678") + run_command("remove").tap do |output| - assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop")}/, output + assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output end @@ -84,7 +103,7 @@ class CliAppTest < CliTestCase test "exec with reuse" do run_command("exec", "--reuse", "ruby -v").tap do |output| - assert_match "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", output # Get current version + assert_match "docker ps --filter label=service=app --filter status=running --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"", output # Get current version assert_match "docker exec app-web-999 ruby -v", output end end @@ -103,28 +122,28 @@ class CliAppTest < CliTestCase test "logs" do SSHKit::Backend::Abstract.any_instance.stubs(:exec) - .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 2>&1'") + .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest| xargs docker logs --timestamps --tail 10 2>&1'") - assert_match "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --tail 100 2>&1", run_command("logs") + assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --tail 100 2>&1", run_command("logs") end test "logs with follow" do SSHKit::Backend::Abstract.any_instance.stubs(:exec) - .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1'") + .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1'") - assert_match "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow") + assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow") end test "version" do run_command("version").tap do |output| - assert_match "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", output + assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"", output end end test "version through main" do stdouted { Mrsk::Cli::Main.start(["app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1"]) }.tap do |output| - assert_match "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", output + assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"", output end end diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index b27e5630..8e1d7545 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -17,6 +17,7 @@ class CliMainTest < CliTestCase Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options) + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop_stale_containers", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options) @@ -26,6 +27,7 @@ class CliMainTest < CliTestCase assert_match /Build and push app image/, output assert_match /Ensure Traefik is running/, output assert_match /Ensure app can pass healthcheck/, output + assert_match /Stop stale containers/, output assert_match /Prune old containers and images/, output end end @@ -38,6 +40,7 @@ class CliMainTest < CliTestCase Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options) + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop_stale_containers", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options) @@ -48,6 +51,7 @@ class CliMainTest < CliTestCase assert_match /Pull app image/, output assert_match /Ensure Traefik is running/, output assert_match /Ensure app can pass healthcheck/, output + assert_match /Stop stale containers/, output assert_match /Prune old containers and images/, output assert_match /Releasing the deploy lock/, output end @@ -58,6 +62,7 @@ class CliMainTest < CliTestCase Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options) + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop_stale_containers", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options) run_command("redeploy").tap do |output| @@ -71,6 +76,7 @@ class CliMainTest < CliTestCase Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options) + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop_stale_containers", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options) run_command("redeploy", "--skip_push").tap do |output| @@ -90,7 +96,7 @@ class CliMainTest < CliTestCase test "rollback good version" do Mrsk::Cli::Main.any_instance.stubs(:container_name_available?).returns(true) - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1").returns("version-to-rollback\n").at_least_once + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"").returns("version-to-rollback\n").at_least_once run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output| assert_match "Start version 123", output @@ -101,7 +107,7 @@ class CliMainTest < CliTestCase test "rollback without old version" do Mrsk::Cli::Main.any_instance.stubs(:container_name_available?).returns(true) - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1").returns("").at_least_once + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"").returns("").at_least_once run_command("rollback", "123").tap do |output| assert_match "Start version 123", output @@ -224,12 +230,19 @@ class CliMainTest < CliTestCase end test "remove with confirmation" do + %w[ web workers ].each do |role| + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"") + .times(2) + .returns("12345678") + end + run_command("remove", "-y", config_file: "deploy_with_accessories").tap do |output| assert_match /docker container stop traefik/, output assert_match /docker container prune --force --filter label=org.opencontainers.image.title=Traefik/, output assert_match /docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik/, output - assert_match /docker ps --quiet --filter label=service=app | xargs docker stop/, output + assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output assert_match /docker container prune --force --filter label=service=app/, output assert_match /docker image prune --all --force --filter label=service=app/, output diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index 75a55ae3..75a25f2f 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -63,14 +63,14 @@ class CommandsAppTest < ActiveSupport::TestCase test "stop" do assert_equal \ - "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop", + "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop", new_command.stop.join(" ") end test "stop with custom stop wait time" do @config[:stop_wait_time] = 30 assert_equal \ - "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop -t 30", + "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop -t 30", new_command.stop.join(" ") end @@ -96,37 +96,37 @@ class CommandsAppTest < ActiveSupport::TestCase test "logs" do assert_equal \ - "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs 2>&1", + "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs 2>&1", new_command.logs.join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m 2>&1", + "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --since 5m 2>&1", new_command.logs(since: "5m").join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --tail 100 2>&1", + "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --tail 100 2>&1", new_command.logs(lines: "100").join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m --tail 100 2>&1", + "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --since 5m --tail 100 2>&1", new_command.logs(since: "5m", lines: "100").join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs 2>&1 | grep 'my-id'", + "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs 2>&1 | grep 'my-id'", new_command.logs(grep: "my-id").join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m 2>&1 | grep 'my-id'", + "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --since 5m 2>&1 | grep 'my-id'", new_command.logs(since: "5m", grep: "my-id").join(" ") end test "follow logs" do assert_match \ - "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1", + "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1", new_command.follow_logs(host: "app-1") assert_match \ - "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1 | grep \"Completed\"", + "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1 | grep \"Completed\"", new_command.follow_logs(host: "app-1", grep: "Completed") end @@ -178,17 +178,17 @@ class CommandsAppTest < ActiveSupport::TestCase end - test "current_container_id" do + test "current_running_container_id" do assert_equal \ - "docker ps --quiet --filter label=service=app --filter label=role=web", - new_command.current_container_id.join(" ") + "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest", + new_command.current_running_container_id.join(" ") end - test "current_container_id with destination" do + test "current_running_container_id with destination" do @destination = "staging" assert_equal \ - "docker ps --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web", - new_command.current_container_id.join(" ") + "docker ps --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --latest", + new_command.current_running_container_id.join(" ") end test "container_id_for" do @@ -199,10 +199,20 @@ class CommandsAppTest < ActiveSupport::TestCase test "current_running_version" do assert_equal \ - "docker ps --filter label=service=app --filter label=role=web --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", + "docker ps --filter label=service=app --filter label=role=web --filter status=running --latest --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"", new_command.current_running_version.join(" ") end + test "list_versions" do + assert_equal \ + "docker ps --filter label=service=app --filter label=role=web --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"", + new_command.list_versions.join(" ") + + assert_equal \ + "docker ps --filter label=service=app --filter label=role=web --filter status=running --latest --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"", + new_command.list_versions("--latest", status: :running).join(" ") + end + test "list_containers" do assert_equal \ "docker container ls --all --filter label=service=app --filter label=role=web",