Clear stale containers

By stopping all the older containers with matching /#{service}-#{role}-#{dest}-.*/ running on the same host.

lib/mrsk/cli/app.rb

@@ -50,12 +50,32 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   desc "stop", "Stop app container on servers"
   def stop
     with_lock do
-      on(MRSK.hosts) do |host|
+      MRSK.hosts.each do |host|
         roles = MRSK.roles_on(host)
 
         roles.each do |role|
-          execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug
-          execute *MRSK.app(role: role).stop, raise_on_non_zero_exit: false
+          on(host) { execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug }
+          all_versions = list_versions(host: host, role: role)
+          all_versions.each do |version|
+            on(host) { execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false }
+          end
+        end
+      end
+    end
+  end
+
+  desc "stop_stale_containers", "Stop all app stale containers"
+  def stop_stale_containers
+    with_lock do
+      MRSK.hosts.each do |host|
+        roles = MRSK.roles_on(host)
+
+        roles.each do |role|
+          stale_versions = list_versions(host: host, role: role).drop(1)
+          stale_versions.each do |version|
+            say "Stopping stale container with version #{version}", :red
+            on(host) { execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false }
+          end
         end
       end
     end
@@ -235,9 +255,13 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
     end
 
     def current_running_version(host: MRSK.primary_host)
-      version = nil
-      on(host) { version = capture_with_info(*MRSK.app.current_running_version).strip }
-      version.presence
+      list_versions(host: host, status: :running).shift.presence
+    end
+
+    def list_versions(host:, role: nil, status: nil)
+      versions = nil
+      on(host) { versions = capture_with_info(*MRSK.app(role: role).list_versions(status: status)).split("\n").map(&:strip) }
+      versions
     end
 
     def version_or_latest

lib/mrsk/cli/main.rb

@@ -37,6 +37,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
+        say "Stop stale containers...", :magenta
+        invoke "mrsk:cli:app:stop_stale_containers", [], invoke_options
+
         invoke "mrsk:cli:app:boot", [], invoke_options
 
         say "Prune old containers and images...", :magenta
@@ -65,6 +68,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
+        say "Stop stale containers...", :magenta
+        invoke "mrsk:cli:app:stop_stale_containers", [], invoke_options
+
         invoke "mrsk:cli:app:boot", [], invoke_options
       end
 

lib/mrsk/commands/app.rb

@@ -29,7 +29,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
 
   def stop(version: nil)
     pipe \
-      version ? container_id_for_version(version) : current_container_id,
+      version ? container_id_for_version(version) : current_running_container_id,
       xargs(config.stop_wait_time ? docker(:stop, "-t", config.stop_wait_time) : docker(:stop))
   end
 
@@ -40,7 +40,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
 
   def logs(since: nil, lines: nil, grep: nil)
     pipe \
-      current_container_id,
+      current_running_container_id,
       "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
       ("grep '#{grep}'" if grep)
   end
@@ -48,7 +48,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   def follow_logs(host:, grep: nil)
     run_over_ssh \
       pipe(
-        current_container_id,
+        current_running_container_id,
         "xargs docker logs --timestamps --tail 10 --follow 2>&1",
         (%(grep "#{grep}") if grep)
       ),
@@ -82,8 +82,8 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   end
 
 
-  def current_container_id
-    docker :ps, "--quiet", *filter_args
+  def current_running_container_id
+    docker :ps, "--quiet", *filter_args(status: :running), "--latest"
   end
 
   def container_id_for_version(version)
@@ -91,11 +91,13 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   end
 
   def current_running_version
-    # FIXME: Find more graceful way to extract the version from "app-version" than using sed and tail!
+    list_versions("--latest", status: :running)
+  end
+
+  def list_versions(*docker_args, status: nil)
     pipe \
-      docker(:ps, *filter_args, "--format", '"{{.Names}}"'),
-      %(sed 's/-/\\n/g'),
-      "tail -n 1"
+      docker(:ps, *filter_args(status: status), *docker_args, "--format", '"{{.Names}}"'),
+      %(grep -oP "(?<=\-)[^-]+$") # Extract SHA from "service-role-dest-SHA"
   end
 
   def list_containers
@@ -134,14 +136,15 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
       [ config.service, role, config.destination, version || config.version ].compact.join("-")
     end
 
-    def filter_args
-      argumentize "--filter", filters
+    def filter_args(status: nil)
+      argumentize "--filter", filters(status: status)
     end
 
-    def filters
+    def filters(status: nil)
       [ "label=service=#{config.service}" ].tap do |filters|
         filters << "label=destination=#{config.destination}" if config.destination
         filters << "label=role=#{role}" if role
+        filters << "status=#{status}" if status
       end
     end
 end