From e2f6db5caec094cc329d2f2c7b28cd38292157b1 Mon Sep 17 00:00:00 2001
From: Jacopo <beschi.jacopo@gmail.com>
Date: Wed, 5 Apr 2023 19:03:27 +0200
Subject: [PATCH 01/13] Clear stale containers

By stopping all the older containers with matching /#{service}-#{role}-#{dest}-.*/ running on the same host.
---
 lib/mrsk/cli/app.rb       | 36 ++++++++++++++++++++++++++------
 lib/mrsk/cli/main.rb      |  6 ++++++
 lib/mrsk/commands/app.rb  | 27 +++++++++++++-----------
 test/cli/app_test.rb      | 39 +++++++++++++++++++++++++---------
 test/cli/main_test.rb     | 19 ++++++++++++++---
 test/commands/app_test.rb | 44 ++++++++++++++++++++++++---------------
 6 files changed, 123 insertions(+), 48 deletions(-)

diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
index fe2599a1..a9dfb75a 100644
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -50,12 +50,32 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   desc "stop", "Stop app container on servers"
   def stop
     with_lock do
-      on(MRSK.hosts) do |host|
+      MRSK.hosts.each do |host|
         roles = MRSK.roles_on(host)
 
         roles.each do |role|
-          execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug
-          execute *MRSK.app(role: role).stop, raise_on_non_zero_exit: false
+          on(host) { execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug }
+          all_versions = list_versions(host: host, role: role)
+          all_versions.each do |version|
+            on(host) { execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false }
+          end
+        end
+      end
+    end
+  end
+
+  desc "stop_stale_containers", "Stop all app stale containers"
+  def stop_stale_containers
+    with_lock do
+      MRSK.hosts.each do |host|
+        roles = MRSK.roles_on(host)
+
+        roles.each do |role|
+          stale_versions = list_versions(host: host, role: role).drop(1)
+          stale_versions.each do |version|
+            say "Stopping stale container with version #{version}", :red
+            on(host) { execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false }
+          end
         end
       end
     end
@@ -235,9 +255,13 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
     end
 
     def current_running_version(host: MRSK.primary_host)
-      version = nil
-      on(host) { version = capture_with_info(*MRSK.app.current_running_version).strip }
-      version.presence
+      list_versions(host: host, status: :running).shift.presence
+    end
+
+    def list_versions(host:, role: nil, status: nil)
+      versions = nil
+      on(host) { versions = capture_with_info(*MRSK.app(role: role).list_versions(status: status)).split("\n").map(&:strip) }
+      versions
     end
 
     def version_or_latest
diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb
index 83f71c34..ac6a54d6 100644
--- a/lib/mrsk/cli/main.rb
+++ b/lib/mrsk/cli/main.rb
@@ -37,6 +37,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
+        say "Stop stale containers...", :magenta
+        invoke "mrsk:cli:app:stop_stale_containers", [], invoke_options
+
         invoke "mrsk:cli:app:boot", [], invoke_options
 
         say "Prune old containers and images...", :magenta
@@ -65,6 +68,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
+        say "Stop stale containers...", :magenta
+        invoke "mrsk:cli:app:stop_stale_containers", [], invoke_options
+
         invoke "mrsk:cli:app:boot", [], invoke_options
       end
 
diff --git a/lib/mrsk/commands/app.rb b/lib/mrsk/commands/app.rb
index 224e7090..2f5a5657 100644
--- a/lib/mrsk/commands/app.rb
+++ b/lib/mrsk/commands/app.rb
@@ -29,7 +29,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
 
   def stop(version: nil)
     pipe \
-      version ? container_id_for_version(version) : current_container_id,
+      version ? container_id_for_version(version) : current_running_container_id,
       xargs(config.stop_wait_time ? docker(:stop, "-t", config.stop_wait_time) : docker(:stop))
   end
 
@@ -40,7 +40,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
 
   def logs(since: nil, lines: nil, grep: nil)
     pipe \
-      current_container_id,
+      current_running_container_id,
       "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
       ("grep '#{grep}'" if grep)
   end
@@ -48,7 +48,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   def follow_logs(host:, grep: nil)
     run_over_ssh \
       pipe(
-        current_container_id,
+        current_running_container_id,
         "xargs docker logs --timestamps --tail 10 --follow 2>&1",
         (%(grep "#{grep}") if grep)
       ),
@@ -82,8 +82,8 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   end
 
 
-  def current_container_id
-    docker :ps, "--quiet", *filter_args
+  def current_running_container_id
+    docker :ps, "--quiet", *filter_args(status: :running), "--latest"
   end
 
   def container_id_for_version(version)
@@ -91,11 +91,13 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   end
 
   def current_running_version
-    # FIXME: Find more graceful way to extract the version from "app-version" than using sed and tail!
+    list_versions("--latest", status: :running)
+  end
+
+  def list_versions(*docker_args, status: nil)
     pipe \
-      docker(:ps, *filter_args, "--format", '"{{.Names}}"'),
-      %(sed 's/-/\\n/g'),
-      "tail -n 1"
+      docker(:ps, *filter_args(status: status), *docker_args, "--format", '"{{.Names}}"'),
+      %(grep -oP "(?<=\-)[^-]+$") # Extract SHA from "service-role-dest-SHA"
   end
 
   def list_containers
@@ -134,14 +136,15 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
       [ config.service, role, config.destination, version || config.version ].compact.join("-")
     end
 
-    def filter_args
-      argumentize "--filter", filters
+    def filter_args(status: nil)
+      argumentize "--filter", filters(status: status)
     end
 
-    def filters
+    def filters(status: nil)
       [ "label=service=#{config.service}" ].tap do |filters|
         filters << "label=destination=#{config.destination}" if config.destination
         filters << "label=role=#{role}" if role
+        filters << "status=#{status}" if status
       end
     end
 end
diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb
index 2dc5ad64..cb3ca638 100644
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -19,7 +19,7 @@ class CliAppTest < CliTestCase
       .returns("12345678") # running version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1")
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"")
       .returns("123") # old version
 
     run_command("boot").tap do |output|
@@ -39,8 +39,23 @@ class CliAppTest < CliTestCase
   end
 
   test "stop" do
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"")
+      .returns("12345678")
+
     run_command("stop").tap do |output|
-      assert_match "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop", output
+      assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output
+    end
+  end
+
+  test "stop_stale_containers" do
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"")
+      .returns("12345678\n87654321")
+
+    run_command("stop_stale_containers").tap do |output|
+      assert_match /Stopping stale container with version 87654321/, output
+      assert_match /#{Regexp.escape("docker container ls --all --filter name=^app-web-87654321$ --quiet | xargs docker stop")}/, output
     end
   end
 
@@ -51,8 +66,12 @@ class CliAppTest < CliTestCase
   end
 
   test "remove" do
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"")
+      .returns("12345678")
+
     run_command("remove").tap do |output|
-      assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop")}/, output
+      assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output
       assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output
       assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output
     end
@@ -84,7 +103,7 @@ class CliAppTest < CliTestCase
 
   test "exec with reuse" do
     run_command("exec", "--reuse", "ruby -v").tap do |output|
-      assert_match "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", output # Get current version
+      assert_match "docker ps --filter label=service=app --filter status=running --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"", output # Get current version
       assert_match "docker exec app-web-999 ruby -v", output
     end
   end
@@ -103,28 +122,28 @@ class CliAppTest < CliTestCase
 
   test "logs" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 2>&1'")
+      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest| xargs docker logs --timestamps --tail 10 2>&1'")
 
-    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --tail 100 2>&1", run_command("logs")
+    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --tail 100 2>&1", run_command("logs")
   end
 
   test "logs with follow" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
+      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
 
-    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
+    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
   end
 
   test "version" do
     run_command("version").tap do |output|
-      assert_match "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", output
+      assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"", output
     end
   end
 
 
   test "version through main" do
     stdouted { Mrsk::Cli::Main.start(["app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1"]) }.tap do |output|
-      assert_match "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", output
+      assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"", output
     end
   end
 
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index b27e5630..8e1d7545 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -17,6 +17,7 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop_stale_containers", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
 
@@ -26,6 +27,7 @@ class CliMainTest < CliTestCase
       assert_match /Build and push app image/, output
       assert_match /Ensure Traefik is running/, output
       assert_match /Ensure app can pass healthcheck/, output
+      assert_match /Stop stale containers/, output
       assert_match /Prune old containers and images/, output
     end
   end
@@ -38,6 +40,7 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop_stale_containers", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
 
@@ -48,6 +51,7 @@ class CliMainTest < CliTestCase
       assert_match /Pull app image/, output
       assert_match /Ensure Traefik is running/, output
       assert_match /Ensure app can pass healthcheck/, output
+      assert_match /Stop stale containers/, output
       assert_match /Prune old containers and images/, output
       assert_match /Releasing the deploy lock/, output
     end
@@ -58,6 +62,7 @@ class CliMainTest < CliTestCase
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop_stale_containers", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
 
     run_command("redeploy").tap do |output|
@@ -71,6 +76,7 @@ class CliMainTest < CliTestCase
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop_stale_containers", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
 
     run_command("redeploy", "--skip_push").tap do |output|
@@ -90,7 +96,7 @@ class CliMainTest < CliTestCase
 
   test "rollback good version" do
     Mrsk::Cli::Main.any_instance.stubs(:container_name_available?).returns(true)
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1").returns("version-to-rollback\n").at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"").returns("version-to-rollback\n").at_least_once
 
     run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
       assert_match "Start version 123", output
@@ -101,7 +107,7 @@ class CliMainTest < CliTestCase
 
   test "rollback without old version" do
     Mrsk::Cli::Main.any_instance.stubs(:container_name_available?).returns(true)
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1").returns("").at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"").returns("").at_least_once
 
     run_command("rollback", "123").tap do |output|
       assert_match "Start version 123", output
@@ -224,12 +230,19 @@ class CliMainTest < CliTestCase
   end
 
   test "remove with confirmation" do
+    %w[ web workers ].each do |role|
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"")
+        .times(2)
+        .returns("12345678")
+    end
+
     run_command("remove", "-y", config_file: "deploy_with_accessories").tap do |output|
       assert_match /docker container stop traefik/, output
       assert_match /docker container prune --force --filter label=org.opencontainers.image.title=Traefik/, output
       assert_match /docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik/, output
 
-      assert_match /docker ps --quiet --filter label=service=app | xargs docker stop/, output
+      assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output
       assert_match /docker container prune --force --filter label=service=app/, output
       assert_match /docker image prune --all --force --filter label=service=app/, output
 
diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb
index 75a55ae3..75a25f2f 100644
--- a/test/commands/app_test.rb
+++ b/test/commands/app_test.rb
@@ -63,14 +63,14 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "stop" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop",
       new_command.stop.join(" ")
   end
 
   test "stop with custom stop wait time" do
     @config[:stop_wait_time] = 30
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop -t 30",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop -t 30",
       new_command.stop.join(" ")
   end
 
@@ -96,37 +96,37 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "logs" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs 2>&1",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs 2>&1",
       new_command.logs.join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m 2>&1",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --since 5m 2>&1",
       new_command.logs(since: "5m").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --tail 100 2>&1",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --tail 100 2>&1",
       new_command.logs(lines: "100").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m --tail 100 2>&1",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --since 5m --tail 100 2>&1",
       new_command.logs(since: "5m", lines: "100").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs 2>&1 | grep 'my-id'",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs 2>&1 | grep 'my-id'",
       new_command.logs(grep: "my-id").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
       new_command.logs(since: "5m", grep: "my-id").join(" ")
   end
 
   test "follow logs" do
     assert_match \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1",
       new_command.follow_logs(host: "app-1")
 
     assert_match \
-      "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1 | grep \"Completed\"",
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1 | grep \"Completed\"",
       new_command.follow_logs(host: "app-1", grep: "Completed")
   end
 
@@ -178,17 +178,17 @@ class CommandsAppTest < ActiveSupport::TestCase
   end
 
 
-  test "current_container_id" do
+  test "current_running_container_id" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web",
-      new_command.current_container_id.join(" ")
+      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest",
+      new_command.current_running_container_id.join(" ")
   end
 
-  test "current_container_id with destination" do
+  test "current_running_container_id with destination" do
     @destination = "staging"
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web",
-      new_command.current_container_id.join(" ")
+      "docker ps --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --latest",
+      new_command.current_running_container_id.join(" ")
   end
 
   test "container_id_for" do
@@ -199,10 +199,20 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "current_running_version" do
     assert_equal \
-      "docker ps --filter label=service=app --filter label=role=web --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1",
+      "docker ps --filter label=service=app --filter label=role=web --filter status=running --latest --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"",
       new_command.current_running_version.join(" ")
   end
 
+  test "list_versions" do
+    assert_equal \
+      "docker ps --filter label=service=app --filter label=role=web --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"",
+      new_command.list_versions.join(" ")
+
+    assert_equal \
+      "docker ps --filter label=service=app --filter label=role=web --filter status=running --latest --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"",
+      new_command.list_versions("--latest", status: :running).join(" ")
+  end
+
   test "list_containers" do
     assert_equal \
       "docker container ls --all --filter label=service=app --filter label=role=web",

From e980f1164e244f6b333723da1866105036f974a1 Mon Sep 17 00:00:00 2001
From: Jacopo <beschi.jacopo@gmail.com>
Date: Thu, 6 Apr 2023 10:12:35 +0200
Subject: [PATCH 02/13] Avoid using GNU-only Perl Regepx Grep

---
 lib/mrsk/cli/app.rb         |  4 ++--
 lib/mrsk/commands/app.rb    |  3 ++-
 lib/mrsk/sshkit_with_ext.rb |  4 ++--
 test/cli/app_test.rb        | 14 +++++++-------
 test/cli/main_test.rb       |  6 +++---
 test/commands/app_test.rb   |  6 +++---
 6 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
index a9dfb75a..428fb13f 100644
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -22,7 +22,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
                 execute *MRSK.app(role: role).rename_container(version: version, new_version: tmp_version)
               end
 
-              old_version = capture_with_info(*MRSK.app(role: role).current_running_version).strip
+              old_version = capture_with_info(*MRSK.app(role: role).current_running_version, raise_on_non_zero_exit: false).strip
               execute *MRSK.app(role: role).run
               sleep MRSK.config.readiness_delay
               execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
@@ -260,7 +260,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
 
     def list_versions(host:, role: nil, status: nil)
       versions = nil
-      on(host) { versions = capture_with_info(*MRSK.app(role: role).list_versions(status: status)).split("\n").map(&:strip) }
+      on(host) { versions = capture_with_info(*MRSK.app(role: role).list_versions(status: status), raise_on_non_zero_exit: false).split("\n").map(&:strip) }
       versions
     end
 
diff --git a/lib/mrsk/commands/app.rb b/lib/mrsk/commands/app.rb
index 2f5a5657..7f435934 100644
--- a/lib/mrsk/commands/app.rb
+++ b/lib/mrsk/commands/app.rb
@@ -97,7 +97,8 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   def list_versions(*docker_args, status: nil)
     pipe \
       docker(:ps, *filter_args(status: status), *docker_args, "--format", '"{{.Names}}"'),
-      %(grep -oP "(?<=\-)[^-]+$") # Extract SHA from "service-role-dest-SHA"
+      %(grep -oE "\\-[^-]+$"), # Extract SHA from "service-role-dest-SHA"
+      %(cut -c 2-)
   end
 
   def list_containers
diff --git a/lib/mrsk/sshkit_with_ext.rb b/lib/mrsk/sshkit_with_ext.rb
index df63fabe..075c2643 100644
--- a/lib/mrsk/sshkit_with_ext.rb
+++ b/lib/mrsk/sshkit_with_ext.rb
@@ -2,8 +2,8 @@ require "sshkit"
 require "sshkit/dsl"
 
 class SSHKit::Backend::Abstract
-  def capture_with_info(*args)
-    capture(*args, verbosity: Logger::INFO)
+  def capture_with_info(*args, **kwargs)
+    capture(*args, **kwargs, verbosity: Logger::INFO)
   end
 
   def puts_by_host(host, output, type: "App")
diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb
index cb3ca638..3797b238 100644
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -19,7 +19,7 @@ class CliAppTest < CliTestCase
       .returns("12345678") # running version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"")
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
       .returns("123") # old version
 
     run_command("boot").tap do |output|
@@ -40,7 +40,7 @@ class CliAppTest < CliTestCase
 
   test "stop" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"")
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
       .returns("12345678")
 
     run_command("stop").tap do |output|
@@ -50,7 +50,7 @@ class CliAppTest < CliTestCase
 
   test "stop_stale_containers" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"")
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
       .returns("12345678\n87654321")
 
     run_command("stop_stale_containers").tap do |output|
@@ -67,7 +67,7 @@ class CliAppTest < CliTestCase
 
   test "remove" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"")
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
       .returns("12345678")
 
     run_command("remove").tap do |output|
@@ -103,7 +103,7 @@ class CliAppTest < CliTestCase
 
   test "exec with reuse" do
     run_command("exec", "--reuse", "ruby -v").tap do |output|
-      assert_match "docker ps --filter label=service=app --filter status=running --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"", output # Get current version
+      assert_match "docker ps --filter label=service=app --filter status=running --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output # Get current version
       assert_match "docker exec app-web-999 ruby -v", output
     end
   end
@@ -136,14 +136,14 @@ class CliAppTest < CliTestCase
 
   test "version" do
     run_command("version").tap do |output|
-      assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"", output
+      assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output
     end
   end
 
 
   test "version through main" do
     stdouted { Mrsk::Cli::Main.start(["app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1"]) }.tap do |output|
-      assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"", output
+      assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output
     end
   end
 
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index 8e1d7545..3811f2c6 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -96,7 +96,7 @@ class CliMainTest < CliTestCase
 
   test "rollback good version" do
     Mrsk::Cli::Main.any_instance.stubs(:container_name_available?).returns(true)
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"").returns("version-to-rollback\n").at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-").returns("version-to-rollback\n").at_least_once
 
     run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
       assert_match "Start version 123", output
@@ -107,7 +107,7 @@ class CliMainTest < CliTestCase
 
   test "rollback without old version" do
     Mrsk::Cli::Main.any_instance.stubs(:container_name_available?).returns(true)
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"").returns("").at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-").returns("").at_least_once
 
     run_command("rollback", "123").tap do |output|
       assert_match "Start version 123", output
@@ -232,7 +232,7 @@ class CliMainTest < CliTestCase
   test "remove with confirmation" do
     %w[ web workers ].each do |role|
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--format", "\"{{.Names}}\"", "|", "grep -oP \"(?<=-)[^-]+$\"")
+        .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
         .times(2)
         .returns("12345678")
     end
diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb
index 75a25f2f..6727e5ed 100644
--- a/test/commands/app_test.rb
+++ b/test/commands/app_test.rb
@@ -199,17 +199,17 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "current_running_version" do
     assert_equal \
-      "docker ps --filter label=service=app --filter label=role=web --filter status=running --latest --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"",
+      "docker ps --filter label=service=app --filter label=role=web --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-",
       new_command.current_running_version.join(" ")
   end
 
   test "list_versions" do
     assert_equal \
-      "docker ps --filter label=service=app --filter label=role=web --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"",
+      "docker ps --filter label=service=app --filter label=role=web --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-",
       new_command.list_versions.join(" ")
 
     assert_equal \
-      "docker ps --filter label=service=app --filter label=role=web --filter status=running --latest --format \"{{.Names}}\" | grep -oP \"(?<=-)[^-]+$\"",
+      "docker ps --filter label=service=app --filter label=role=web --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-",
       new_command.list_versions("--latest", status: :running).join(" ")
   end
 

From 4d47fbdf4155b1c36c1885932c6771849cf47953 Mon Sep 17 00:00:00 2001
From: Jacopo <beschi.jacopo@gmail.com>
Date: Thu, 6 Apr 2023 10:42:02 +0200
Subject: [PATCH 03/13] Merge stop and stop_stale_containers

---
 lib/mrsk/cli/app.rb   | 28 +++++++---------------------
 lib/mrsk/cli/main.rb  |  8 ++++----
 test/cli/app_test.rb  |  5 ++---
 test/cli/main_test.rb | 12 ++++++------
 4 files changed, 19 insertions(+), 34 deletions(-)

diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
index 428fb13f..641bceb8 100644
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -48,32 +48,18 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   end
 
   desc "stop", "Stop app container on servers"
+  option :only_old, aliases: "-o", type: :boolean, default: false, desc: "Stop only old containers"
   def stop
     with_lock do
       MRSK.hosts.each do |host|
         roles = MRSK.roles_on(host)
 
         roles.each do |role|
-          on(host) { execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug }
-          all_versions = list_versions(host: host, role: role)
-          all_versions.each do |version|
-            on(host) { execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false }
-          end
-        end
-      end
-    end
-  end
+          audit_record = options[:only_old] ? "Stopped old containers" : "Stopped app"
+          on(host) { execute *MRSK.auditor(role: role).record(audit_record), verbosity: :debug }
 
-  desc "stop_stale_containers", "Stop all app stale containers"
-  def stop_stale_containers
-    with_lock do
-      MRSK.hosts.each do |host|
-        roles = MRSK.roles_on(host)
-
-        roles.each do |role|
-          stale_versions = list_versions(host: host, role: role).drop(1)
-          stale_versions.each do |version|
-            say "Stopping stale container with version #{version}", :red
+          versions = list_versions(host: host, role: role, only_old: options[:only_old])
+          versions.each do |version|
             on(host) { execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false }
           end
         end
@@ -258,10 +244,10 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
       list_versions(host: host, status: :running).shift.presence
     end
 
-    def list_versions(host:, role: nil, status: nil)
+    def list_versions(host:, role: nil, status: nil, only_old: false)
       versions = nil
       on(host) { versions = capture_with_info(*MRSK.app(role: role).list_versions(status: status), raise_on_non_zero_exit: false).split("\n").map(&:strip) }
-      versions
+      only_old ? versions.drop(1) : versions
     end
 
     def version_or_latest
diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb
index ac6a54d6..11e77b22 100644
--- a/lib/mrsk/cli/main.rb
+++ b/lib/mrsk/cli/main.rb
@@ -37,8 +37,8 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
-        say "Stop stale containers...", :magenta
-        invoke "mrsk:cli:app:stop_stale_containers", [], invoke_options
+        say "Stop old containers...", :magenta
+        invoke "mrsk:cli:app:stop", [], invoke_options.merge({ "only_old" => true })
 
         invoke "mrsk:cli:app:boot", [], invoke_options
 
@@ -68,8 +68,8 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
-        say "Stop stale containers...", :magenta
-        invoke "mrsk:cli:app:stop_stale_containers", [], invoke_options
+        say "Stop old containers...", :magenta
+        invoke "mrsk:cli:app:stop", [], invoke_options.merge({ "only_old" => true })
 
         invoke "mrsk:cli:app:boot", [], invoke_options
       end
diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb
index 3797b238..597fa294 100644
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -48,13 +48,12 @@ class CliAppTest < CliTestCase
     end
   end
 
-  test "stop_stale_containers" do
+  test "stop only old containers" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
       .returns("12345678\n87654321")
 
-    run_command("stop_stale_containers").tap do |output|
-      assert_match /Stopping stale container with version 87654321/, output
+    run_command("stop", '--only-old').tap do |output|
       assert_match /#{Regexp.escape("docker container ls --all --filter name=^app-web-87654321$ --quiet | xargs docker stop")}/, output
     end
   end
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index 3811f2c6..e396439a 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -17,7 +17,7 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop_stale_containers", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop", [], invoke_options.merge({ "only_old" => true }))
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
 
@@ -27,7 +27,7 @@ class CliMainTest < CliTestCase
       assert_match /Build and push app image/, output
       assert_match /Ensure Traefik is running/, output
       assert_match /Ensure app can pass healthcheck/, output
-      assert_match /Stop stale containers/, output
+      assert_match /Stop old containers/, output
       assert_match /Prune old containers and images/, output
     end
   end
@@ -40,7 +40,7 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop_stale_containers", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop", [], invoke_options.merge({ "only_old" => true }))
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
 
@@ -51,7 +51,7 @@ class CliMainTest < CliTestCase
       assert_match /Pull app image/, output
       assert_match /Ensure Traefik is running/, output
       assert_match /Ensure app can pass healthcheck/, output
-      assert_match /Stop stale containers/, output
+      assert_match /Stop old containers/, output
       assert_match /Prune old containers and images/, output
       assert_match /Releasing the deploy lock/, output
     end
@@ -62,7 +62,7 @@ class CliMainTest < CliTestCase
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop_stale_containers", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop", [], invoke_options.merge({ "only_old" => true }))
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
 
     run_command("redeploy").tap do |output|
@@ -76,7 +76,7 @@ class CliMainTest < CliTestCase
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop_stale_containers", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop", [], invoke_options.merge({ "only_old" => true }))
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
 
     run_command("redeploy", "--skip_push").tap do |output|

From 8ae5331d97f58c633f2887f9a28401ce6d2880f9 Mon Sep 17 00:00:00 2001
From: Jacopo <beschi.jacopo@gmail.com>
Date: Thu, 6 Apr 2023 11:26:15 +0200
Subject: [PATCH 04/13] Boot stop all the old containers

---
 lib/mrsk/cli/app.rb   | 16 +++++++++++-----
 lib/mrsk/cli/main.rb  |  6 ------
 test/cli/app_test.rb  | 17 +++++++++++------
 test/cli/main_test.rb |  6 ------
 4 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
index 641bceb8..9c98ba6b 100644
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -8,13 +8,13 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
 
         cli = self
 
-        on(MRSK.hosts) do |host|
+        MRSK.hosts.each do |host|
           roles = MRSK.roles_on(host)
 
           roles.each do |role|
-            execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug
+            on(host) do
+              execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug
 
-            begin
               if capture_with_info(*MRSK.app(role: role).container_id_for_version(version)).present?
                 tmp_version = "#{version}_#{SecureRandom.hex(8)}"
                 info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
@@ -22,10 +22,16 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
                 execute *MRSK.app(role: role).rename_container(version: version, new_version: tmp_version)
               end
 
-              old_version = capture_with_info(*MRSK.app(role: role).current_running_version, raise_on_non_zero_exit: false).strip
               execute *MRSK.app(role: role).run
               sleep MRSK.config.readiness_delay
-              execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
+            end
+
+            old_versions = list_versions(host: host, role: role, only_old: true)
+            old_versions.each do |old_version|
+              on(host) do
+                info "Stopping old container with version #{old_version}"
+                execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false
+              end
             end
           end
         end
diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb
index 11e77b22..83f71c34 100644
--- a/lib/mrsk/cli/main.rb
+++ b/lib/mrsk/cli/main.rb
@@ -37,9 +37,6 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
-        say "Stop old containers...", :magenta
-        invoke "mrsk:cli:app:stop", [], invoke_options.merge({ "only_old" => true })
-
         invoke "mrsk:cli:app:boot", [], invoke_options
 
         say "Prune old containers and images...", :magenta
@@ -68,9 +65,6 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
-        say "Stop old containers...", :magenta
-        invoke "mrsk:cli:app:stop", [], invoke_options.merge({ "only_old" => true })
-
         invoke "mrsk:cli:app:boot", [], invoke_options
       end
 
diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb
index 597fa294..245b7246 100644
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -2,8 +2,13 @@ require_relative "cli_test_case"
 
 class CliAppTest < CliTestCase
   test "boot" do
-    # Stub current version fetch
-    SSHKit::Backend::Abstract.any_instance.stubs(:capture).returns("123") # old version
+    # current version not running yet
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
+      .returns("124\n123") # running + old version
 
     run_command("boot").tap do |output|
       assert_match "docker run --detach --restart unless-stopped", output
@@ -16,17 +21,17 @@ class CliAppTest < CliTestCase
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet")
-      .returns("12345678") # running version
+      .returns("12345678") # current version already running
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
-      .returns("123") # old version
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
+      .returns("123\n12345678_1") # running + renamed version
 
     run_command("boot").tap do |output|
       assert_match /Renaming container .* to .* as already deployed on 1.1.1.1/, output # Rename
       assert_match /docker rename .* .*/, output
       assert_match "docker run --detach --restart unless-stopped", output
-      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+      assert_match "docker container ls --all --filter name=^app-web-12345678_1$ --quiet | xargs docker stop", output
     end
   ensure
     Thread.report_on_exception = true
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index e396439a..112e9f67 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -17,7 +17,6 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop", [], invoke_options.merge({ "only_old" => true }))
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
 
@@ -27,7 +26,6 @@ class CliMainTest < CliTestCase
       assert_match /Build and push app image/, output
       assert_match /Ensure Traefik is running/, output
       assert_match /Ensure app can pass healthcheck/, output
-      assert_match /Stop old containers/, output
       assert_match /Prune old containers and images/, output
     end
   end
@@ -40,7 +38,6 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop", [], invoke_options.merge({ "only_old" => true }))
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
 
@@ -51,7 +48,6 @@ class CliMainTest < CliTestCase
       assert_match /Pull app image/, output
       assert_match /Ensure Traefik is running/, output
       assert_match /Ensure app can pass healthcheck/, output
-      assert_match /Stop old containers/, output
       assert_match /Prune old containers and images/, output
       assert_match /Releasing the deploy lock/, output
     end
@@ -62,7 +58,6 @@ class CliMainTest < CliTestCase
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop", [], invoke_options.merge({ "only_old" => true }))
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
 
     run_command("redeploy").tap do |output|
@@ -76,7 +71,6 @@ class CliMainTest < CliTestCase
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
-    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stop", [], invoke_options.merge({ "only_old" => true }))
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
 
     run_command("redeploy", "--skip_push").tap do |output|

From f9436d567340e0fa3b197e9d19ee7e1ebd7be68a Mon Sep 17 00:00:00 2001
From: Jacopo <beschi.jacopo@gmail.com>
Date: Thu, 6 Apr 2023 11:39:12 +0200
Subject: [PATCH 05/13] Style

---
 lib/mrsk/cli/app.rb      | 2 +-
 lib/mrsk/commands/app.rb | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
index 9c98ba6b..2ac92213 100644
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -54,7 +54,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   end
 
   desc "stop", "Stop app container on servers"
-  option :only_old, aliases: "-o", type: :boolean, default: false, desc: "Stop only old containers"
+  option :only_old, aliases: "-o", type: :boolean, default: false, desc: "Stop only old versions containers"
   def stop
     with_lock do
       MRSK.hosts.each do |host|
diff --git a/lib/mrsk/commands/app.rb b/lib/mrsk/commands/app.rb
index 7f435934..fc192038 100644
--- a/lib/mrsk/commands/app.rb
+++ b/lib/mrsk/commands/app.rb
@@ -97,7 +97,8 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   def list_versions(*docker_args, status: nil)
     pipe \
       docker(:ps, *filter_args(status: status), *docker_args, "--format", '"{{.Names}}"'),
-      %(grep -oE "\\-[^-]+$"), # Extract SHA from "service-role-dest-SHA"
+      # Extract SHA from "service-role-dest-SHA"
+      %(grep -oE "\\-[^-]+$"), 
       %(cut -c 2-)
   end
 

From 579b4cd9aa351b3f96818c219535793ff50a2b26 Mon Sep 17 00:00:00 2001
From: Jacopo <beschi.jacopo@gmail.com>
Date: Tue, 11 Apr 2023 10:22:03 +0200
Subject: [PATCH 06/13] Simplify

By using and ad-hoc command to detect and stop stale containers.
By default stale containers are only detected.
---
 lib/mrsk/cli/app.rb      | 67 ++++++++++++++++++++++++----------------
 lib/mrsk/cli/main.rb     |  6 ++++
 lib/mrsk/commands/app.rb |  3 +-
 test/cli/app_test.rb     | 46 +++++++++++++--------------
 test/cli/main_test.rb    | 15 +++++----
 5 files changed, 76 insertions(+), 61 deletions(-)

diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
index 2ac92213..b8aa7195 100644
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -8,13 +8,13 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
 
         cli = self
 
-        MRSK.hosts.each do |host|
+        on(MRSK.hosts) do |host|
           roles = MRSK.roles_on(host)
 
           roles.each do |role|
-            on(host) do
-              execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug
+            execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug
 
+            begin
               if capture_with_info(*MRSK.app(role: role).container_id_for_version(version)).present?
                 tmp_version = "#{version}_#{SecureRandom.hex(8)}"
                 info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
@@ -22,16 +22,10 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
                 execute *MRSK.app(role: role).rename_container(version: version, new_version: tmp_version)
               end
 
+              old_version = capture_with_info(*MRSK.app(role: role).current_running_version, raise_on_non_zero_exit: false).strip
               execute *MRSK.app(role: role).run
               sleep MRSK.config.readiness_delay
-            end
-
-            old_versions = list_versions(host: host, role: role, only_old: true)
-            old_versions.each do |old_version|
-              on(host) do
-                info "Stopping old container with version #{old_version}"
-                execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false
-              end
+              execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
             end
           end
         end
@@ -54,20 +48,14 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   end
 
   desc "stop", "Stop app container on servers"
-  option :only_old, aliases: "-o", type: :boolean, default: false, desc: "Stop only old versions containers"
   def stop
     with_lock do
-      MRSK.hosts.each do |host|
+      on(MRSK.hosts) do |host|
         roles = MRSK.roles_on(host)
 
         roles.each do |role|
-          audit_record = options[:only_old] ? "Stopped old containers" : "Stopped app"
-          on(host) { execute *MRSK.auditor(role: role).record(audit_record), verbosity: :debug }
-
-          versions = list_versions(host: host, role: role, only_old: options[:only_old])
-          versions.each do |version|
-            on(host) { execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false }
-          end
+          execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug
+          execute *MRSK.app(role: role).stop, raise_on_non_zero_exit: false
         end
       end
     end
@@ -136,6 +124,35 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
     on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_containers) }
   end
 
+  desc "stale_containers", "Detect app stale containers"
+  option :stop, aliases: "-s", type: :boolean, default: false, desc: "Stop the stale containers found."
+  def stale_containers
+    with_lock do
+      stop = options[:stop]
+
+      on(MRSK.hosts) do |host|
+        roles = MRSK.roles_on(host)
+
+        roles.each do |role|
+          stale_versions = \
+            capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false)
+            .split("\n")
+            .map(&:strip)
+            .drop(1)
+
+          stale_versions.each do |version|
+            if stop
+              puts_by_host host, "Stopping stale container with version #{version}"
+              execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false
+            else
+              puts_by_host host,  "Detected stale container with version #{version} (use `mrsk app stale_containers --stop` to stop)"
+            end
+          end
+        end
+      end
+    end
+  end
+
   desc "images", "Show app images on servers"
   def images
     on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_images) }
@@ -247,13 +264,9 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
     end
 
     def current_running_version(host: MRSK.primary_host)
-      list_versions(host: host, status: :running).shift.presence
-    end
-
-    def list_versions(host:, role: nil, status: nil, only_old: false)
-      versions = nil
-      on(host) { versions = capture_with_info(*MRSK.app(role: role).list_versions(status: status), raise_on_non_zero_exit: false).split("\n").map(&:strip) }
-      only_old ? versions.drop(1) : versions
+      version = nil
+      on(host) { version = capture_with_info(*MRSK.app.current_running_version).strip }
+      version.presence
     end
 
     def version_or_latest
diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb
index 83f71c34..e39f923c 100644
--- a/lib/mrsk/cli/main.rb
+++ b/lib/mrsk/cli/main.rb
@@ -37,6 +37,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
+        say "Detect stale containers...", :magenta
+        invoke "mrsk:cli:app:stale_containers", [], invoke_options
+
         invoke "mrsk:cli:app:boot", [], invoke_options
 
         say "Prune old containers and images...", :magenta
@@ -65,6 +68,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
+        say "Detect stale containers...", :magenta
+        invoke "mrsk:cli:app:stale_containers", [], invoke_options
+
         invoke "mrsk:cli:app:boot", [], invoke_options
       end
 
diff --git a/lib/mrsk/commands/app.rb b/lib/mrsk/commands/app.rb
index fc192038..7f435934 100644
--- a/lib/mrsk/commands/app.rb
+++ b/lib/mrsk/commands/app.rb
@@ -97,8 +97,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   def list_versions(*docker_args, status: nil)
     pipe \
       docker(:ps, *filter_args(status: status), *docker_args, "--format", '"{{.Names}}"'),
-      # Extract SHA from "service-role-dest-SHA"
-      %(grep -oE "\\-[^-]+$"), 
+      %(grep -oE "\\-[^-]+$"), # Extract SHA from "service-role-dest-SHA"
       %(cut -c 2-)
   end
 
diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb
index 245b7246..53925717 100644
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -2,13 +2,8 @@ require_relative "cli_test_case"
 
 class CliAppTest < CliTestCase
   test "boot" do
-    # current version not running yet
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet")
-
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
-      .returns("124\n123") # running + old version
+    # Stub current version fetch
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture).returns("123") # old version
 
     run_command("boot").tap do |output|
       assert_match "docker run --detach --restart unless-stopped", output
@@ -21,17 +16,17 @@ class CliAppTest < CliTestCase
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet")
-      .returns("12345678") # current version already running
+      .returns("12345678") # running version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
-      .returns("123\n12345678_1") # running + renamed version
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
+      .returns("123") # old version
 
     run_command("boot").tap do |output|
       assert_match /Renaming container .* to .* as already deployed on 1.1.1.1/, output # Rename
       assert_match /docker rename .* .*/, output
       assert_match "docker run --detach --restart unless-stopped", output
-      assert_match "docker container ls --all --filter name=^app-web-12345678_1$ --quiet | xargs docker stop", output
+      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
     end
   ensure
     Thread.report_on_exception = true
@@ -44,21 +39,28 @@ class CliAppTest < CliTestCase
   end
 
   test "stop" do
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
-      .returns("12345678")
-
     run_command("stop").tap do |output|
-      assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output
+      assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop", output
     end
   end
 
-  test "stop only old containers" do
+  test "stale_containers" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
       .returns("12345678\n87654321")
 
-    run_command("stop", '--only-old').tap do |output|
+    run_command("stale_containers").tap do |output|
+      assert_match /Detected stale container with version 87654321/, output
+    end
+  end
+
+  test "stop stale_containers" do
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
+      .returns("12345678\n87654321")
+
+    run_command("stale_containers", "--stop").tap do |output|
+      assert_match /Stopping stale container with version 87654321/, output
       assert_match /#{Regexp.escape("docker container ls --all --filter name=^app-web-87654321$ --quiet | xargs docker stop")}/, output
     end
   end
@@ -70,12 +72,8 @@ class CliAppTest < CliTestCase
   end
 
   test "remove" do
-    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
-      .returns("12345678")
-
     run_command("remove").tap do |output|
-      assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output
+      assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop")}/, output
       assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output
       assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output
     end
@@ -107,7 +105,7 @@ class CliAppTest < CliTestCase
 
   test "exec with reuse" do
     run_command("exec", "--reuse", "ruby -v").tap do |output|
-      assert_match "docker ps --filter label=service=app --filter status=running --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output # Get current version
+      assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output # Get current version
       assert_match "docker exec app-web-999 ruby -v", output
     end
   end
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index 112e9f67..289fe866 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -17,6 +17,7 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
 
@@ -26,6 +27,7 @@ class CliMainTest < CliTestCase
       assert_match /Build and push app image/, output
       assert_match /Ensure Traefik is running/, output
       assert_match /Ensure app can pass healthcheck/, output
+      assert_match /Detect stale containers/, output
       assert_match /Prune old containers and images/, output
     end
   end
@@ -38,6 +40,7 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
 
@@ -48,6 +51,7 @@ class CliMainTest < CliTestCase
       assert_match /Pull app image/, output
       assert_match /Ensure Traefik is running/, output
       assert_match /Ensure app can pass healthcheck/, output
+      assert_match /Detect stale containers/, output
       assert_match /Prune old containers and images/, output
       assert_match /Releasing the deploy lock/, output
     end
@@ -58,6 +62,7 @@ class CliMainTest < CliTestCase
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
 
     run_command("redeploy").tap do |output|
@@ -71,6 +76,7 @@ class CliMainTest < CliTestCase
 
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
+    Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
 
     run_command("redeploy", "--skip_push").tap do |output|
@@ -224,19 +230,12 @@ class CliMainTest < CliTestCase
   end
 
   test "remove with confirmation" do
-    %w[ web workers ].each do |role|
-      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
-        .times(2)
-        .returns("12345678")
-    end
-
     run_command("remove", "-y", config_file: "deploy_with_accessories").tap do |output|
       assert_match /docker container stop traefik/, output
       assert_match /docker container prune --force --filter label=org.opencontainers.image.title=Traefik/, output
       assert_match /docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik/, output
 
-      assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output
+      assert_match /docker ps --quiet --filter label=service=app | xargs docker stop/, output
       assert_match /docker container prune --force --filter label=service=app/, output
       assert_match /docker image prune --all --force --filter label=service=app/, output
 

From 03d933d10bff8e698ec9eedb8bf6180a55fc65a0 Mon Sep 17 00:00:00 2001
From: Jacopo <beschi.jacopo@gmail.com>
Date: Tue, 11 Apr 2023 10:59:25 +0200
Subject: [PATCH 07/13] Add Role to the message

---
 lib/mrsk/cli/app.rb  | 4 ++--
 test/cli/app_test.rb | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
index b8aa7195..0c7733d9 100644
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -142,10 +142,10 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
 
           stale_versions.each do |version|
             if stop
-              puts_by_host host, "Stopping stale container with version #{version}"
+              puts_by_host host, "Stopping stale container for role #{role} with version #{version}"
               execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false
             else
-              puts_by_host host,  "Detected stale container with version #{version} (use `mrsk app stale_containers --stop` to stop)"
+              puts_by_host host,  "Detected stale container for role #{role} with version #{version} (use `mrsk app stale_containers --stop` to stop)"
             end
           end
         end
diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb
index 53925717..c3001fd9 100644
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -50,7 +50,7 @@ class CliAppTest < CliTestCase
       .returns("12345678\n87654321")
 
     run_command("stale_containers").tap do |output|
-      assert_match /Detected stale container with version 87654321/, output
+      assert_match /Detected stale container for role web with version 87654321/, output
     end
   end
 
@@ -60,7 +60,7 @@ class CliAppTest < CliTestCase
       .returns("12345678\n87654321")
 
     run_command("stale_containers", "--stop").tap do |output|
-      assert_match /Stopping stale container with version 87654321/, output
+      assert_match /Stopping stale container for role web with version 87654321/, output
       assert_match /#{Regexp.escape("docker container ls --all --filter name=^app-web-87654321$ --quiet | xargs docker stop")}/, output
     end
   end

From c3393c8213f81d2fe7387b2c41d38b4d3328e87b Mon Sep 17 00:00:00 2001
From: Jacopo <beschi.jacopo@gmail.com>
Date: Tue, 11 Apr 2023 11:03:11 +0200
Subject: [PATCH 08/13] Remove dot

---
 lib/mrsk/cli/app.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
index 0c7733d9..ed9c19e2 100644
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -125,7 +125,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   end
 
   desc "stale_containers", "Detect app stale containers"
-  option :stop, aliases: "-s", type: :boolean, default: false, desc: "Stop the stale containers found."
+  option :stop, aliases: "-s", type: :boolean, default: false, desc: "Stop the stale containers found"
   def stale_containers
     with_lock do
       stop = options[:stop]

From 12e3a562c4cae631045a4952ce6c197d40af93f7 Mon Sep 17 00:00:00 2001
From: Jacopo <beschi.jacopo@gmail.com>
Date: Tue, 11 Apr 2023 15:24:49 +0200
Subject: [PATCH 09/13] Extract helper

---
 lib/mrsk/cli/app.rb         |  8 +-------
 lib/mrsk/sshkit_with_ext.rb | 10 ++++++++++
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
index ed9c19e2..efd0dc38 100644
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -134,13 +134,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
         roles = MRSK.roles_on(host)
 
         roles.each do |role|
-          stale_versions = \
-            capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false)
-            .split("\n")
-            .map(&:strip)
-            .drop(1)
-
-          stale_versions.each do |version|
+          stale_versions(role: role).each do |version|
             if stop
               puts_by_host host, "Stopping stale container for role #{role} with version #{version}"
               execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false
diff --git a/lib/mrsk/sshkit_with_ext.rb b/lib/mrsk/sshkit_with_ext.rb
index 075c2643..d657a07c 100644
--- a/lib/mrsk/sshkit_with_ext.rb
+++ b/lib/mrsk/sshkit_with_ext.rb
@@ -1,7 +1,17 @@
 require "sshkit"
 require "sshkit/dsl"
 
+module AppHelper
+  def stale_versions(role:)
+    capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false)
+      .split("\n")
+      .drop(1)
+  end
+end
+
 class SSHKit::Backend::Abstract
+  include AppHelper
+
   def capture_with_info(*args, **kwargs)
     capture(*args, **kwargs, verbosity: Logger::INFO)
   end

From 0bfd4ca7806ebf887727bc2190740c704372bbd9 Mon Sep 17 00:00:00 2001
From: Jacopo <beschi.jacopo@gmail.com>
Date: Tue, 11 Apr 2023 16:04:46 +0200
Subject: [PATCH 10/13] Use cli = self approach

---
 lib/mrsk/cli/app.rb         | 15 ++++++++++++++-
 lib/mrsk/sshkit_with_ext.rb | 10 ----------
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
index efd0dc38..70787cdd 100644
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -130,11 +130,13 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
     with_lock do
       stop = options[:stop]
 
+      cli = self
+
       on(MRSK.hosts) do |host|
         roles = MRSK.roles_on(host)
 
         roles.each do |role|
-          stale_versions(role: role).each do |version|
+          cli.stale_versions(host: host, role: role).each do |version|
             if stop
               puts_by_host host, "Stopping stale container for role #{role} with version #{version}"
               execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false
@@ -242,6 +244,17 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
     on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.current_running_version).strip }
   end
 
+  def stale_versions(host:, role:)
+    stale_versions = nil
+    on(host) do
+      stale_versions = \
+        capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false)
+        .split("\n")
+        .drop(1)
+    end
+    stale_versions
+  end
+
   private
     def using_version(new_version)
       if new_version
diff --git a/lib/mrsk/sshkit_with_ext.rb b/lib/mrsk/sshkit_with_ext.rb
index d657a07c..075c2643 100644
--- a/lib/mrsk/sshkit_with_ext.rb
+++ b/lib/mrsk/sshkit_with_ext.rb
@@ -1,17 +1,7 @@
 require "sshkit"
 require "sshkit/dsl"
 
-module AppHelper
-  def stale_versions(role:)
-    capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false)
-      .split("\n")
-      .drop(1)
-  end
-end
-
 class SSHKit::Backend::Abstract
-  include AppHelper
-
   def capture_with_info(*args, **kwargs)
     capture(*args, **kwargs, verbosity: Logger::INFO)
   end

From e8697327fa0beeff4cbe7d0f4e834320d6248a37 Mon Sep 17 00:00:00 2001
From: Jacopo <beschi.jacopo@gmail.com>
Date: Tue, 11 Apr 2023 16:20:16 +0200
Subject: [PATCH 11/13] Use no_commands block

---
 lib/mrsk/cli/app.rb | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
index 70787cdd..f8c21960 100644
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -244,15 +244,17 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
     on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.current_running_version).strip }
   end
 
-  def stale_versions(host:, role:)
-    stale_versions = nil
-    on(host) do
-      stale_versions = \
-        capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false)
-        .split("\n")
-        .drop(1)
+  no_commands do
+    def stale_versions(host:, role:)
+      stale_versions = nil
+      on(host) do
+        stale_versions = \
+          capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false)
+          .split("\n")
+          .drop(1)
+      end
+      stale_versions
     end
-    stale_versions
   end
 
   private

From 72b70e3e9e2baeb08fe7b96e5571ee8ff763bc73 Mon Sep 17 00:00:00 2001
From: Jacopo <beschi.jacopo@gmail.com>
Date: Tue, 11 Apr 2023 16:22:47 +0200
Subject: [PATCH 12/13] More compact

---
 lib/mrsk/cli/app.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
index f8c21960..69c0bfa2 100644
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -246,14 +246,14 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
 
   no_commands do
     def stale_versions(host:, role:)
-      stale_versions = nil
+      versions = nil
       on(host) do
-        stale_versions = \
+        versions = \
           capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false)
           .split("\n")
           .drop(1)
       end
-      stale_versions
+      versions
     end
   end
 

From 3cbf4aea46de08f4ff5ebfc6a35d52e8a60796df Mon Sep 17 00:00:00 2001
From: Jacopo <beschi.jacopo@gmail.com>
Date: Wed, 12 Apr 2023 11:53:49 +0200
Subject: [PATCH 13/13] Make method private method and use :send

---
 lib/mrsk/cli/app.rb | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
index 69c0bfa2..2c3e8a40 100644
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -136,7 +136,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
         roles = MRSK.roles_on(host)
 
         roles.each do |role|
-          cli.stale_versions(host: host, role: role).each do |version|
+          cli.send(:stale_versions, host: host, role: role).each do |version|
             if stop
               puts_by_host host, "Stopping stale container for role #{role} with version #{version}"
               execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false
@@ -244,19 +244,6 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
     on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.current_running_version).strip }
   end
 
-  no_commands do
-    def stale_versions(host:, role:)
-      versions = nil
-      on(host) do
-        versions = \
-          capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false)
-          .split("\n")
-          .drop(1)
-      end
-      versions
-    end
-  end
-
   private
     def using_version(new_version)
       if new_version
@@ -278,6 +265,17 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
       version.presence
     end
 
+    def stale_versions(host:, role:)
+      versions = nil
+      on(host) do
+        versions = \
+          capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false)
+          .split("\n")
+          .drop(1)
+      end
+      versions
+    end
+
     def version_or_latest
       options[:version] || "latest"
     end