Merge pull request #183 from basecamp/cleanup-excessive-containers-running

Clear stale containers

lib/mrsk/cli/app.rb

@@ -19,14 +19,14 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
           roles.each do |role|
             execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug
 
-            if capture_with_info(*MRSK.app(role: role).container_id_for_version(version)).present?
+            if capture_with_info(*MRSK.app(role: role).container_id_for_version(version), raise_on_non_zero_exit: false).present?
               tmp_version = "#{version}_#{SecureRandom.hex(8)}"
               info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
               execute *MRSK.auditor(role: role).record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
               execute *MRSK.app(role: role).rename_container(version: version, new_version: tmp_version)
             end
 
-            old_version = capture_with_info(*MRSK.app(role: role).current_running_version).strip
+            old_version = capture_with_info(*MRSK.app(role: role).current_running_version, raise_on_non_zero_exit: false).strip
             execute *MRSK.app(role: role).run
             sleep MRSK.config.readiness_delay
             execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
@@ -127,6 +127,31 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
     on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_containers) }
   end
 
+  desc "stale_containers", "Detect app stale containers"
+  option :stop, aliases: "-s", type: :boolean, default: false, desc: "Stop the stale containers found"
+  def stale_containers
+    with_lock do
+      stop = options[:stop]
+
+      cli = self
+
+      on(MRSK.hosts) do |host|
+        roles = MRSK.roles_on(host)
+
+        roles.each do |role|
+          cli.send(:stale_versions, host: host, role: role).each do |version|
+            if stop
+              puts_by_host host, "Stopping stale container for role #{role} with version #{version}"
+              execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false
+            else
+              puts_by_host host,  "Detected stale container for role #{role} with version #{version} (use `mrsk app stale_containers --stop` to stop)"
+            end
+          end
+        end
+      end
+    end
+  end
+
   desc "images", "Show app images on servers"
   def images
     on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_images) }
@@ -243,6 +268,17 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
       version.presence
     end
 
+    def stale_versions(host:, role:)
+      versions = nil
+      on(host) do
+        versions = \
+          capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false)
+          .split("\n")
+          .drop(1)
+      end
+      versions
+    end
+
     def version_or_latest
       options[:version] || "latest"
     end

lib/mrsk/cli/main.rb

@@ -37,6 +37,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
+        say "Detect stale containers...", :magenta
+        invoke "mrsk:cli:app:stale_containers", [], invoke_options
+
         hold_lock_on_error do
           invoke "mrsk:cli:app:boot", [], invoke_options
         end
@@ -67,6 +70,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
+        say "Detect stale containers...", :magenta
+        invoke "mrsk:cli:app:stale_containers", [], invoke_options
+
         hold_lock_on_error do
           invoke "mrsk:cli:app:boot", [], invoke_options
         end

lib/mrsk/commands/app.rb

@@ -29,7 +29,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
 
   def stop(version: nil)
     pipe \
-      version ? container_id_for_version(version) : current_container_id,
+      version ? container_id_for_version(version) : current_running_container_id,
       xargs(config.stop_wait_time ? docker(:stop, "-t", config.stop_wait_time) : docker(:stop))
   end
 
@@ -40,7 +40,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
 
   def logs(since: nil, lines: nil, grep: nil)
     pipe \
-      current_container_id,
+      current_running_container_id,
       "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
       ("grep '#{grep}'" if grep)
   end
@@ -48,7 +48,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   def follow_logs(host:, grep: nil)
     run_over_ssh \
       pipe(
-        current_container_id,
+        current_running_container_id,
         "xargs docker logs --timestamps --tail 10 --follow 2>&1",
         (%(grep "#{grep}") if grep)
       ),
@@ -82,8 +82,8 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   end
 
 
-  def current_container_id
-    docker :ps, "--quiet", *filter_args
+  def current_running_container_id
+    docker :ps, "--quiet", *filter_args(status: :running), "--latest"
   end
 
   def container_id_for_version(version)
@@ -91,11 +91,14 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   end
 
   def current_running_version
-    # FIXME: Find more graceful way to extract the version from "app-version" than using sed and tail!
+    list_versions("--latest", status: :running)
+  end
+
+  def list_versions(*docker_args, status: nil)
     pipe \
-      docker(:ps, *filter_args, "--format", '"{{.Names}}"'),
-      %(sed 's/-/\\n/g'),
-      "tail -n 1"
+      docker(:ps, *filter_args(status: status), *docker_args, "--format", '"{{.Names}}"'),
+      %(grep -oE "\\-[^-]+$"), # Extract SHA from "service-role-dest-SHA"
+      %(cut -c 2-)
   end
 
   def list_containers
@@ -138,14 +141,15 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
       [ config.service, role, config.destination, version || config.version ].compact.join("-")
     end
 
-    def filter_args
-      argumentize "--filter", filters
+    def filter_args(status: nil)
+      argumentize "--filter", filters(status: status)
     end
 
-    def filters
+    def filters(status: nil)
       [ "label=service=#{config.service}" ].tap do |filters|
         filters << "label=destination=#{config.destination}" if config.destination
         filters << "label=role=#{role}" if role
+        filters << "status=#{status}" if status
       end
     end
 end

lib/mrsk/sshkit_with_ext.rb

@@ -2,8 +2,8 @@ require "sshkit"
 require "sshkit/dsl"
 
 class SSHKit::Backend::Abstract
-  def capture_with_info(*args)
-    capture(*args, verbosity: Logger::INFO)
+  def capture_with_info(*args, **kwargs)
+    capture(*args, **kwargs, verbosity: Logger::INFO)
   end
 
   def puts_by_host(host, output, type: "App")