spacebar/kamal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify

Browse Source 
By using and ad-hoc command to detect and stop stale containers.
By default stale containers are only detected.
This commit is contained in:
Jacopo
2023-04-11 10:22:03 +02:00
parent f9436d5673
commit 579b4cd9aa
5 changed files with 76 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
lib/mrsk/cli/app.rb
View File

@@ -8,13 +8,13 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
cli = self
MRSK.hosts.each do |host|
on(MRSK.hosts) do |host|
roles = MRSK.roles_on(host)
roles.each do |role|
on(host) do
execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug
begin
if capture_with_info(*MRSK.app(role: role).container_id_for_version(version)).present?
tmp_version = "#{version}_#{SecureRandom.hex(8)}"
info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
@@ -22,16 +22,10 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
execute *MRSK.app(role: role).rename_container(version: version, new_version: tmp_version)
end
old_version = capture_with_info(*MRSK.app(role: role).current_running_version, raise_on_non_zero_exit: false).strip
execute *MRSK.app(role: role).run
sleep MRSK.config.readiness_delay
end
old_versions = list_versions(host: host, role: role, only_old: true)
old_versions.each do |old_version|
on(host) do
info "Stopping old container with version #{old_version}"
execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false
end
execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
end
end
end
@@ -54,20 +48,14 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
end
desc "stop", "Stop app container on servers"
option :only_old, aliases: "-o", type: :boolean, default: false, desc: "Stop only old versions containers"
def stop
with_lock do
MRSK.hosts.each do |host|
on(MRSK.hosts) do |host|
roles = MRSK.roles_on(host)
roles.each do |role|
audit_record = options[:only_old] ? "Stopped old containers" : "Stopped app"
on(host) { execute *MRSK.auditor(role: role).record(audit_record), verbosity: :debug }
versions = list_versions(host: host, role: role, only_old: options[:only_old])
versions.each do |version|
on(host) { execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false }
end
execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug
execute *MRSK.app(role: role).stop, raise_on_non_zero_exit: false
end
end
end
@@ -136,6 +124,35 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_containers) }
end
desc "stale_containers", "Detect app stale containers"
option :stop, aliases: "-s", type: :boolean, default: false, desc: "Stop the stale containers found."
def stale_containers
with_lock do
stop = options[:stop]
on(MRSK.hosts) do |host|
roles = MRSK.roles_on(host)
roles.each do |role|
stale_versions = \
capture_with_info(*MRSK.app(role: role).list_versions, raise_on_non_zero_exit: false)
.split("\n")
.map(&:strip)
.drop(1)
stale_versions.each do |version|
if stop
puts_by_host host, "Stopping stale container with version #{version}"
execute *MRSK.app(role: role).stop(version: version), raise_on_non_zero_exit: false
else
puts_by_host host, "Detected stale container with version #{version} (use `mrsk app stale_containers --stop` to stop)"
end
end
end
end
end
end
desc "images", "Show app images on servers"
def images
on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_images) }
@@ -247,13 +264,9 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
end
def current_running_version(host: MRSK.primary_host)
list_versions(host: host, status: :running).shift.presence
end
def list_versions(host:, role: nil, status: nil, only_old: false)
versions = nil
on(host) { versions = capture_with_info(*MRSK.app(role: role).list_versions(status: status), raise_on_non_zero_exit: false).split("\n").map(&:strip) }
only_old ? versions.drop(1) : versions
version = nil
on(host) { version = capture_with_info(*MRSK.app.current_running_version).strip }
version.presence
end
def version_or_latest

6
lib/mrsk/cli/main.rb
View File

@@ -37,6 +37,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
say "Ensure app can pass healthcheck...", :magenta
invoke "mrsk:cli:healthcheck:perform", [], invoke_options
say "Detect stale containers...", :magenta
invoke "mrsk:cli:app:stale_containers", [], invoke_options
invoke "mrsk:cli:app:boot", [], invoke_options
say "Prune old containers and images...", :magenta
@@ -65,6 +68,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
say "Ensure app can pass healthcheck...", :magenta
invoke "mrsk:cli:healthcheck:perform", [], invoke_options
say "Detect stale containers...", :magenta
invoke "mrsk:cli:app:stale_containers", [], invoke_options
invoke "mrsk:cli:app:boot", [], invoke_options
end

3
lib/mrsk/commands/app.rb
View File

@@ -97,8 +97,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
def list_versions(*docker_args, status: nil)
pipe \
docker(:ps, *filter_args(status: status), *docker_args, "--format", '"{{.Names}}"'),
# Extract SHA from "service-role-dest-SHA"
%(grep -oE "\\-[^-]+$"),
%(grep -oE "\\-[^-]+$"), # Extract SHA from "service-role-dest-SHA"
%(cut -c 2-)
end

46
test/cli/app_test.rb
View File

@@ -2,13 +2,8 @@ require_relative "cli_test_case"
class CliAppTest < CliTestCase
test "boot" do
# current version not running yet
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet")
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
.returns("124\n123") # running + old version
# Stub current version fetch
SSHKit::Backend::Abstract.any_instance.stubs(:capture).returns("123") # old version
run_command("boot").tap do |output|
assert_match "docker run --detach --restart unless-stopped", output
@@ -21,17 +16,17 @@ class CliAppTest < CliTestCase
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet")
.returns("12345678") # current version already running
.returns("12345678") # running version
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
.returns("123\n12345678_1") # running + renamed version
.with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
.returns("123") # old version
run_command("boot").tap do |output|
assert_match /Renaming container .* to .* as already deployed on 1.1.1.1/, output # Rename
assert_match /docker rename .* .*/, output
assert_match "docker run --detach --restart unless-stopped", output
assert_match "docker container ls --all --filter name=^app-web-12345678_1$ --quiet | xargs docker stop", output
assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
end
ensure
Thread.report_on_exception = true
@@ -44,21 +39,28 @@ class CliAppTest < CliTestCase
end
test "stop" do
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
.returns("12345678")
run_command("stop").tap do |output|
assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output
assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop", output
end
end
test "stop only old containers" do
test "stale_containers" do
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
.returns("12345678\n87654321")
run_command("stop", '--only-old').tap do |output|
run_command("stale_containers").tap do |output|
assert_match /Detected stale container with version 87654321/, output
end
end
test "stop stale_containers" do
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
.returns("12345678\n87654321")
run_command("stale_containers", "--stop").tap do |output|
assert_match /Stopping stale container with version 87654321/, output
assert_match /#{Regexp.escape("docker container ls --all --filter name=^app-web-87654321$ --quiet | xargs docker stop")}/, output
end
end
@@ -70,12 +72,8 @@ class CliAppTest < CliTestCase
end
test "remove" do
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
.returns("12345678")
run_command("remove").tap do |output|
assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output
assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop")}/, output
assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output
assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output
end
@@ -107,7 +105,7 @@ class CliAppTest < CliTestCase
test "exec with reuse" do
run_command("exec", "--reuse", "ruby -v").tap do |output|
assert_match "docker ps --filter label=service=app --filter status=running --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output # Get current version
assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output # Get current version
assert_match "docker exec app-web-999 ruby -v", output
end
end

15
test/cli/main_test.rb
View File

@@ -17,6 +17,7 @@ class CliMainTest < CliTestCase
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
@@ -26,6 +27,7 @@ class CliMainTest < CliTestCase
assert_match /Build and push app image/, output
assert_match /Ensure Traefik is running/, output
assert_match /Ensure app can pass healthcheck/, output
assert_match /Detect stale containers/, output
assert_match /Prune old containers and images/, output
end
end
@@ -38,6 +40,7 @@ class CliMainTest < CliTestCase
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
@@ -48,6 +51,7 @@ class CliMainTest < CliTestCase
assert_match /Pull app image/, output
assert_match /Ensure Traefik is running/, output
assert_match /Ensure app can pass healthcheck/, output
assert_match /Detect stale containers/, output
assert_match /Prune old containers and images/, output
assert_match /Releasing the deploy lock/, output
end
@@ -58,6 +62,7 @@ class CliMainTest < CliTestCase
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
run_command("redeploy").tap do |output|
@@ -71,6 +76,7 @@ class CliMainTest < CliTestCase
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
run_command("redeploy", "--skip_push").tap do |output|
@@ -224,19 +230,12 @@ class CliMainTest < CliTestCase
end
test "remove with confirmation" do
%w[ web workers ].each do |role|
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
.times(2)
.returns("12345678")
end
run_command("remove", "-y", config_file: "deploy_with_accessories").tap do |output|
assert_match /docker container stop traefik/, output
assert_match /docker container prune --force --filter label=org.opencontainers.image.title=Traefik/, output
assert_match /docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik/, output
assert_match "docker container ls --all --filter name=^app-web-12345678$ --quiet | xargs docker stop", output
assert_match /docker ps --quiet --filter label=service=app | xargs docker stop/, output
assert_match /docker container prune --force --filter label=service=app/, output
assert_match /docker image prune --all --force --filter label=service=app/, output