From 55dd2f49c13ba1ba0de09803756cfcfa9cf6fd1a Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Wed, 27 Mar 2024 16:25:39 +0000
Subject: [PATCH 1/6] Tag image after booting and include destination

If you are deploying more than one destination to a host, the latest
tags will conflict, so we'll append the destination to the tag.

The latest tag is used when booting the app or exec-ing a new container.

If a deploy doesn't complete on a host for all roles then we should
probably not be using it, so move the tagging to the end of the boot
process.
---
 lib/kamal/cli/app.rb             | 6 +++---
 lib/kamal/commands/app/assets.rb | 2 +-
 lib/kamal/commands/app/images.rb | 2 +-
 lib/kamal/configuration.rb       | 2 +-
 test/commands/app_test.rb        | 8 +-------
 5 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/lib/kamal/cli/app.rb b/lib/kamal/cli/app.rb
index 278bd1af..93f31dd8 100644
--- a/lib/kamal/cli/app.rb
+++ b/lib/kamal/cli/app.rb
@@ -9,9 +9,6 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
           # Assets are prepared in a separate step to ensure they are on all hosts before booting
           on(KAMAL.hosts) do
-            execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
-            execute *KAMAL.app.tag_current_image_as_latest
-
             KAMAL.roles_on(host).each do |role|
               Kamal::Cli::App::PrepareAssets.new(host, role, self).run
             end
@@ -21,6 +18,9 @@ class Kamal::Cli::App < Kamal::Cli::Base
             KAMAL.roles_on(host).each do |role|
               Kamal::Cli::App::Boot.new(host, role, version, self).run
             end
+
+            execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
+            execute *KAMAL.app.tag_latest_image
           end
         end
       end
diff --git a/lib/kamal/commands/app/assets.rb b/lib/kamal/commands/app/assets.rb
index dba651b2..26ca6e0c 100644
--- a/lib/kamal/commands/app/assets.rb
+++ b/lib/kamal/commands/app/assets.rb
@@ -5,7 +5,7 @@ module Kamal::Commands::App::Assets
     combine \
       make_directory(role.asset_extracted_path),
       [ *docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true" ],
-      docker(:run, "--name", asset_container, "--detach", "--rm", config.latest_image, "sleep 1000000"),
+      docker(:run, "--name", asset_container, "--detach", "--rm", config.absolute_image, "sleep 1000000"),
       docker(:cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_path),
       docker(:stop, "-t 1", asset_container),
       by: "&&"
diff --git a/lib/kamal/commands/app/images.rb b/lib/kamal/commands/app/images.rb
index 9b1ae0b8..e20e83e1 100644
--- a/lib/kamal/commands/app/images.rb
+++ b/lib/kamal/commands/app/images.rb
@@ -7,7 +7,7 @@ module Kamal::Commands::App::Images
     docker :image, :prune, "--all", "--force", *filter_args
   end
 
-  def tag_current_image_as_latest
+  def tag_latest_image
     docker :tag, config.absolute_image, config.latest_image
   end
 end
diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb
index 483ca770..75148d55 100644
--- a/lib/kamal/configuration.rb
+++ b/lib/kamal/configuration.rb
@@ -128,7 +128,7 @@ class Kamal::Configuration
   end
 
   def latest_image
-    "#{repository}:latest"
+    "#{repository}:#{[ "latest", *destination ].join("-")}"
   end
 
   def service_with_version
diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb
index a15e0e7c..ceec9405 100644
--- a/test/commands/app_test.rb
+++ b/test/commands/app_test.rb
@@ -339,12 +339,6 @@ class CommandsAppTest < ActiveSupport::TestCase
       new_command.remove_images.join(" ")
   end
 
-  test "tag_current_image_as_latest" do
-    assert_equal \
-      "docker tag dhh/app:999 dhh/app:latest",
-      new_command.tag_current_image_as_latest.join(" ")
-  end
-
   test "make_env_directory" do
     assert_equal "mkdir -p .kamal/env/roles", new_command.make_env_directory.join(" ")
   end
@@ -371,7 +365,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     assert_equal [
       :mkdir, "-p", ".kamal/assets/extracted/app-web-999", "&&",
       :docker, :stop, "-t 1", "app-web-assets", "2> /dev/null", "|| true", "&&",
-      :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "dhh/app:latest", "sleep 1000000", "&&",
+      :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "dhh/app:999", "sleep 1000000", "&&",
       :docker, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/assets/extracted/app-web-999", "&&",
       :docker, :stop, "-t 1", "app-web-assets"
     ], new_command(asset_path: "/public/assets").extract_assets

From bade195e93fa0ef19b406bec5a3ddc5c6e118588 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Thu, 28 Mar 2024 11:21:24 +0000
Subject: [PATCH 2/6] Redefine what the "latest" container means

Currently the latest container is the one that was created last. But if
we have had a failed deployment that left two containers running that
would not be the one we want. The second container could be in a
restart loop for example.

Instead we want the container that is running the image tagged as
latest. As we now tag as latest after a successful deployment we can
trust that that is a healthy container.

In the case that there is no container running the latest image tag,
we'll fall back to the latest container.

This could happen if the deploy was halted in between the old container
being stopped and the image being tagged as latest.
---
 lib/kamal/commands/app.rb  | 44 +++++++++++++++++++++++++++++++++++---
 lib/kamal/commands/base.rb |  2 +-
 test/cli/app_test.rb       | 24 ++++++++++-----------
 test/cli/main_test.rb      |  4 ++--
 test/commands/app_test.rb  | 38 ++++++++++++++++----------------
 5 files changed, 75 insertions(+), 37 deletions(-)

diff --git a/lib/kamal/commands/app.rb b/lib/kamal/commands/app.rb
index 77e0e0dd..5c0b423b 100644
--- a/lib/kamal/commands/app.rb
+++ b/lib/kamal/commands/app.rb
@@ -49,7 +49,9 @@ class Kamal::Commands::App < Kamal::Commands::Base
 
 
   def current_running_container_id
-    docker :ps, "--quiet", *filter_args(statuses: ACTIVE_DOCKER_STATUSES), "--latest"
+    pipe \
+      [ shell(chain(latest_image_container_id, latest_container_id)) ],
+      [ :head, "-1" ]
   end
 
   def container_id_for_version(version, only_running: false)
@@ -57,13 +59,16 @@ class Kamal::Commands::App < Kamal::Commands::Base
   end
 
   def current_running_version
-    list_versions("--latest", statuses: ACTIVE_DOCKER_STATUSES)
+    pipe \
+      [ shell(chain(latest_image_container_name, latest_container_name)) ],
+      [ :head, "-1" ],
+      extract_version_from_name
   end
 
   def list_versions(*docker_args, statuses: nil)
     pipe \
       docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
-      %(while read line; do echo ${line##{role.container_prefix}-}; done) # Extract SHA from "service-role-dest-SHA"
+      extract_version_from_name
   end
 
 
@@ -81,10 +86,43 @@ class Kamal::Commands::App < Kamal::Commands::Base
       [ role.container_prefix, version || config.version ].compact.join("-")
     end
 
+    def latest_image_id
+      docker :image, :ls, *argumentize("--filter", "reference=#{config.latest_image}"), "--format", "'{{.ID}}'"
+    end
+
+    def latest_image_container_id
+      latest_image_container format: "--quiet"
+    end
+
+    def latest_container_id
+      latest_container format: "--quiet"
+    end
+
+    def latest_image_container_name
+      latest_image_container format: "--format '{{.Names}}'"
+    end
+
+    def latest_container_name
+      latest_container format: "--format '{{.Names}}'"
+    end
+
+    def latest_image_container(format: nil)
+      docker :ps, "--latest", *format, *filter_args(statuses: ACTIVE_DOCKER_STATUSES), "--filter", "ancestor=$(#{latest_image_id.join(" ")})"
+    end
+
+    def latest_container(format:)
+      docker :ps, "--latest", *format, *filter_args(statuses: ACTIVE_DOCKER_STATUSES)
+    end
+
     def filter_args(statuses: nil)
       argumentize "--filter", filters(statuses: statuses)
     end
 
+    def extract_version_from_name
+      # Extract SHA from "service-role-dest-SHA"
+      %(while read line; do echo ${line##{role.container_prefix}-}; done)
+    end
+
     def filters(statuses: nil)
       [ "label=service=#{config.service}" ].tap do |filters|
         filters << "label=destination=#{config.destination}" if config.destination
diff --git a/lib/kamal/commands/base.rb b/lib/kamal/commands/base.rb
index 27948434..8c289ae0 100644
--- a/lib/kamal/commands/base.rb
+++ b/lib/kamal/commands/base.rb
@@ -71,7 +71,7 @@ module Kamal::Commands
       end
 
       def shell(command)
-        [ :sh, "-c", "'#{command.flatten.join(" ").gsub("'", "'\\''")}'" ]
+        [ :sh, "-c", "'#{command.flatten.join(" ").gsub("'", "'\\\\''")}'" ]
       end
 
       def docker(*args)
diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb
index 416bd895..25c3266e 100644
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -23,7 +23,7 @@ class CliAppTest < CliTestCase
       .returns("running") # health check
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
       .returns("123") # old version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -75,7 +75,7 @@ class CliAppTest < CliTestCase
       .returns("running") # health check
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
       .returns("123").twice # old version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -100,7 +100,7 @@ class CliAppTest < CliTestCase
 
   test "stop" do
     run_command("stop").tap do |output|
-      assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop", output
+      assert_match "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop", output
     end
   end
 
@@ -133,7 +133,7 @@ class CliAppTest < CliTestCase
 
   test "remove" do
     run_command("remove").tap do |output|
-      assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop")}/, output
+      assert_match /#{Regexp.escape("sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop")}/, output
       assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output
       assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output
     end
@@ -165,7 +165,7 @@ class CliAppTest < CliTestCase
 
   test "exec with reuse" do
     run_command("exec", "--reuse", "ruby -v").tap do |output|
-      assert_match "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done", output # Get current version
+      assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output # Get current version
       assert_match "docker exec app-web-999 ruby -v", output
     end
   end
@@ -184,7 +184,7 @@ class CliAppTest < CliTestCase
       .with("ssh -t root@1.1.1.1 -p 22 'docker exec -it app-web-999 ruby -v'")
     run_command("exec", "-i", "--reuse", "ruby -v").tap do |output|
       assert_match "Get current version of running container...", output
-      assert_match "Running docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done on 1.1.1.1", output
+      assert_match "Running /usr/bin/env sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done on 1.1.1.1", output
       assert_match "Launching interactive command with version 999 via SSH from existing container on 1.1.1.1...", output
     end
   end
@@ -203,28 +203,28 @@ class CliAppTest < CliTestCase
 
   test "logs" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest| xargs docker logs --timestamps --tail 10 2>&1'")
+      .with("ssh -t root@1.1.1.1 'sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1| xargs docker logs --timestamps --tail 10 2>&1'")
 
-    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --tail 100 2>&1", run_command("logs")
+    assert_match "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --tail 100 2>&1", run_command("logs")
   end
 
   test "logs with follow" do
     SSHKit::Backend::Abstract.any_instance.stubs(:exec)
-      .with("ssh -t root@1.1.1.1 -p 22 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
+      .with("ssh -t root@1.1.1.1 -p 22 'sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
 
-    assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
+    assert_match "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
   end
 
   test "version" do
     run_command("version").tap do |output|
-      assert_match "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done", output
+      assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output
     end
   end
 
 
   test "version through main" do
     stdouted { Kamal::Cli::Main.start([ "app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1" ]) }.tap do |output|
-      assert_match "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done", output
+      assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output
     end
   end
 
diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb
index 6111ade8..6bd9dbcc 100644
--- a/test/cli/main_test.rb
+++ b/test/cli/main_test.rb
@@ -250,7 +250,7 @@ class CliMainTest < CliTestCase
         .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet")
         .returns("version-to-rollback\n").at_least_once
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-        .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-#{role}-}; done", raise_on_non_zero_exit: false)
+        .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=#{role} --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=#{role} --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-#{role}-}; done", raise_on_non_zero_exit: false)
         .returns("version-to-rollback\n").at_least_once
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
         .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
@@ -286,7 +286,7 @@ class CliMainTest < CliTestCase
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
       .returns("").at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
       .returns("").at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb
index ceec9405..e5df6717 100644
--- a/test/commands/app_test.rb
+++ b/test/commands/app_test.rb
@@ -95,14 +95,14 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "stop" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop",
       new_command.stop.join(" ")
   end
 
   test "stop with custom stop wait time" do
     @config[:stop_wait_time] = 30
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker stop -t 30",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker stop -t 30",
       new_command.stop.join(" ")
   end
 
@@ -128,45 +128,45 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "logs" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs 2>&1",
       new_command.logs.join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --since 5m 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --since 5m 2>&1",
       new_command.logs(since: "5m").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --tail 100 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --tail 100 2>&1",
       new_command.logs(lines: "100").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --since 5m --tail 100 2>&1",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --since 5m --tail 100 2>&1",
       new_command.logs(since: "5m", lines: "100").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs 2>&1 | grep 'my-id'",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs 2>&1 | grep 'my-id'",
       new_command.logs(grep: "my-id").join(" ")
 
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
       new_command.logs(since: "5m", grep: "my-id").join(" ")
   end
 
   test "follow logs" do
-    assert_match \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --follow 2>&1",
+    assert_equal \
+      "ssh -t root@app-1 -p 22 'sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --timestamps --follow 2>&1'",
       new_command.follow_logs(host: "app-1")
 
-    assert_match \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --follow 2>&1 | grep \"Completed\"",
+    assert_equal \
+      "ssh -t root@app-1 -p 22 'sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --timestamps --follow 2>&1 | grep \"Completed\"'",
       new_command.follow_logs(host: "app-1", grep: "Completed")
 
-    assert_match \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 123 --follow 2>&1",
+    assert_equal \
+      "ssh -t root@app-1 -p 22 'sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --timestamps --tail 123 --follow 2>&1'",
       new_command.follow_logs(host: "app-1", lines: 123)
 
-    assert_match \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest | xargs docker logs --timestamps --tail 123 --follow 2>&1 | grep \"Completed\"",
+    assert_equal \
+      "ssh -t root@app-1 -p 22 'sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --timestamps --tail 123 --follow 2>&1 | grep \"Completed\"'",
       new_command.follow_logs(host: "app-1", lines: 123, grep: "Completed")
   end
 
@@ -242,14 +242,14 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "current_running_container_id" do
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest",
+    "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1",
       new_command.current_running_container_id.join(" ")
   end
 
   test "current_running_container_id with destination" do
     @destination = "staging"
     assert_equal \
-      "docker ps --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --filter status=restarting --latest",
+      "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest-staging --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --filter status=restarting' | head -1",
       new_command.current_running_container_id.join(" ")
   end
 
@@ -261,7 +261,7 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "current_running_version" do
     assert_equal \
-      "docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done",
+      "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done",
       new_command.current_running_version.join(" ")
   end
 

From fb7d9077ff698fc30b1c17f12170bc0ca5f68bcd Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Fri, 29 Mar 2024 09:26:36 +0000
Subject: [PATCH 3/6] Use latest tag for the current destination

---
 lib/kamal/cli/app.rb                | 2 +-
 lib/kamal/commands/app.rb           | 4 ++--
 lib/kamal/configuration.rb          | 6 +++++-
 test/integration/docker-compose.yml | 2 +-
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/kamal/cli/app.rb b/lib/kamal/cli/app.rb
index 93f31dd8..b4de978d 100644
--- a/lib/kamal/cli/app.rb
+++ b/lib/kamal/cli/app.rb
@@ -284,6 +284,6 @@ class Kamal::Cli::App < Kamal::Cli::Base
     end
 
     def version_or_latest
-      options[:version] || "latest"
+      options[:version] || KAMAL.config.latest_tag
     end
 end
diff --git a/lib/kamal/commands/app.rb b/lib/kamal/commands/app.rb
index 5c0b423b..12d35bc6 100644
--- a/lib/kamal/commands/app.rb
+++ b/lib/kamal/commands/app.rb
@@ -50,7 +50,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
 
   def current_running_container_id
     pipe \
-      [ shell(chain(latest_image_container_id, latest_container_id)) ],
+      shell(chain(latest_image_container_id, latest_container_id)),
       [ :head, "-1" ]
   end
 
@@ -60,7 +60,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
 
   def current_running_version
     pipe \
-      [ shell(chain(latest_image_container_name, latest_container_name)) ],
+      shell(chain(latest_image_container_name, latest_container_name)),
       [ :head, "-1" ],
       extract_version_from_name
   end
diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb
index 75148d55..12aefa78 100644
--- a/lib/kamal/configuration.rb
+++ b/lib/kamal/configuration.rb
@@ -128,7 +128,11 @@ class Kamal::Configuration
   end
 
   def latest_image
-    "#{repository}:#{[ "latest", *destination ].join("-")}"
+    "#{repository}:#{latest_tag}"
+  end
+
+  def latest_tag
+    [ "latest", *destination ].join("-")
   end
 
   def service_with_version
diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml
index da67f164..5b2c31dc 100644
--- a/test/integration/docker-compose.yml
+++ b/test/integration/docker-compose.yml
@@ -18,7 +18,7 @@ services:
     build:
       context: docker/deployer
     environment:
-      - TEST_ID=${TEST_ID}
+      - TEST_ID=${TEST_ID:-}
     volumes:
       - ../..:/kamal
       - shared:/shared

From 05ac808f2afb81ded623adbb9e5370843ba8fc8c Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Fri, 29 Mar 2024 10:23:50 +0000
Subject: [PATCH 4/6] Use image tag to determine stale containers

Use current_running_version to determine the latest version when finding
stale containers.
---
 lib/kamal/cli/app.rb      | 16 ++++------------
 lib/kamal/commands/app.rb | 33 ++++++++++-----------------------
 test/cli/app_test.rb      | 12 ++++++++++--
 3 files changed, 24 insertions(+), 37 deletions(-)

diff --git a/lib/kamal/cli/app.rb b/lib/kamal/cli/app.rb
index b4de978d..3ce88a1a 100644
--- a/lib/kamal/cli/app.rb
+++ b/lib/kamal/cli/app.rb
@@ -136,7 +136,10 @@ class Kamal::Cli::App < Kamal::Cli::Base
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
-          cli.send(:stale_versions, host: host, role: role).each do |version|
+          versions = capture_with_info(*KAMAL.app(role: role).list_versions, raise_on_non_zero_exit: false).split("\n")
+          versions -= [ capture_with_info(*KAMAL.app(role: role).current_running_version, raise_on_non_zero_exit: false).strip ]
+
+          versions.each do |version|
             if stop
               puts_by_host host, "Stopping stale container for role #{role} with version #{version}"
               execute *KAMAL.app(role: role).stop(version: version), raise_on_non_zero_exit: false
@@ -272,17 +275,6 @@ class Kamal::Cli::App < Kamal::Cli::Base
       version.presence
     end
 
-    def stale_versions(host:, role:)
-      versions = nil
-      on(host) do
-        versions = \
-          capture_with_info(*KAMAL.app(role: role).list_versions, raise_on_non_zero_exit: false)
-          .split("\n")
-          .drop(1)
-      end
-      versions
-    end
-
     def version_or_latest
       options[:version] || KAMAL.config.latest_tag
     end
diff --git a/lib/kamal/commands/app.rb b/lib/kamal/commands/app.rb
index 12d35bc6..15992640 100644
--- a/lib/kamal/commands/app.rb
+++ b/lib/kamal/commands/app.rb
@@ -49,9 +49,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
 
 
   def current_running_container_id
-    pipe \
-      shell(chain(latest_image_container_id, latest_container_id)),
-      [ :head, "-1" ]
+    current_running_container(format: "--quiet")
   end
 
   def container_id_for_version(version, only_running: false)
@@ -60,8 +58,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
 
   def current_running_version
     pipe \
-      shell(chain(latest_image_container_name, latest_container_name)),
-      [ :head, "-1" ],
+      current_running_container(format: "--format '{{.Names}}'"),
       extract_version_from_name
   end
 
@@ -90,28 +87,18 @@ class Kamal::Commands::App < Kamal::Commands::Base
       docker :image, :ls, *argumentize("--filter", "reference=#{config.latest_image}"), "--format", "'{{.ID}}'"
     end
 
-    def latest_image_container_id
-      latest_image_container format: "--quiet"
+    def current_running_container(format:)
+      pipe \
+        shell(chain(latest_image_container(format: format), latest_container(format: format))),
+        [ :head, "-1" ]
     end
 
-    def latest_container_id
-      latest_container format: "--quiet"
+    def latest_image_container(format:)
+      latest_container format: format, filters: [ "ancestor=$(#{latest_image_id.join(" ")})" ]
     end
 
-    def latest_image_container_name
-      latest_image_container format: "--format '{{.Names}}'"
-    end
-
-    def latest_container_name
-      latest_container format: "--format '{{.Names}}'"
-    end
-
-    def latest_image_container(format: nil)
-      docker :ps, "--latest", *format, *filter_args(statuses: ACTIVE_DOCKER_STATUSES), "--filter", "ancestor=$(#{latest_image_id.join(" ")})"
-    end
-
-    def latest_container(format:)
-      docker :ps, "--latest", *format, *filter_args(statuses: ACTIVE_DOCKER_STATUSES)
+    def latest_container(format:, filters: nil)
+      docker :ps, "--latest", *format, *filter_args(statuses: ACTIVE_DOCKER_STATUSES), argumentize("--filter", filters)
     end
 
     def filter_args(statuses: nil)
diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb
index 25c3266e..ab0eff51 100644
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -107,7 +107,11 @@ class CliAppTest < CliTestCase
   test "stale_containers" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
-      .returns("12345678\n87654321")
+      .returns("12345678\n87654321\n")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("12345678\n")
 
     run_command("stale_containers").tap do |output|
       assert_match /Detected stale container for role web with version 87654321/, output
@@ -117,7 +121,11 @@ class CliAppTest < CliTestCase
   test "stop stale_containers" do
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
-      .returns("12345678\n87654321")
+      .returns("12345678\n87654321\n")
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("12345678\n")
 
     run_command("stale_containers", "--stop").tap do |output|
       assert_match /Stopping stale container for role web with version 87654321/, output

From ba7a13f895e76af80bc868729d6de3103988f911 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Fri, 29 Mar 2024 10:29:58 +0000
Subject: [PATCH 5/6] Only tag after deploying to all hosts

---
 lib/kamal/cli/app.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/kamal/cli/app.rb b/lib/kamal/cli/app.rb
index 3ce88a1a..2bf2cea7 100644
--- a/lib/kamal/cli/app.rb
+++ b/lib/kamal/cli/app.rb
@@ -18,7 +18,9 @@ class Kamal::Cli::App < Kamal::Cli::Base
             KAMAL.roles_on(host).each do |role|
               Kamal::Cli::App::Boot.new(host, role, version, self).run
             end
+          end
 
+          on(KAMAL.hosts) do |host|
             execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
             execute *KAMAL.app.tag_latest_image
           end

From ed90b99f0de99e03999f8e8f38330dce9113e87e Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Fri, 29 Mar 2024 10:51:57 +0000
Subject: [PATCH 6/6] Add tag_latest_image tests

---
 test/commands/app_test.rb | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb
index e5df6717..4c891124 100644
--- a/test/commands/app_test.rb
+++ b/test/commands/app_test.rb
@@ -339,6 +339,19 @@ class CommandsAppTest < ActiveSupport::TestCase
       new_command.remove_images.join(" ")
   end
 
+  test "tag_latest_image" do
+    assert_equal \
+      "docker tag dhh/app:999 dhh/app:latest",
+      new_command.tag_latest_image.join(" ")
+  end
+
+  test "tag_latest_image with destination" do
+    @destination = "staging"
+    assert_equal \
+      "docker tag dhh/app:999 dhh/app:latest-staging",
+      new_command.tag_latest_image.join(" ")
+  end
+
   test "make_env_directory" do
     assert_equal "mkdir -p .kamal/env/roles", new_command.make_env_directory.join(" ")
   end