spacebar/kamal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #183 from basecamp/cleanup-excessive-containers-running

Browse Source 
Clear stale containers
This commit is contained in:
David Heinemeier Hansson
2023-04-12 15:58:59 +02:00
committed by GitHub
parent d4a72da9d8 9ddb181f50
commit bc8875e020
7 changed files with 131 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
test/cli/app_test.rb
View File

@@ -16,11 +16,11 @@ class CliAppTest < CliTestCase
run_command("details") # Preheat MRSK const
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet")
.with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
.returns("12345678") # running version
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1")
.with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
.returns("123") # old version
run_command("boot").tap do |output|
@@ -41,7 +41,28 @@ class CliAppTest < CliTestCase
test "stop" do
run_command("stop").tap do |output|
assert_match "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop", output
assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop", output
end
end
test "stale_containers" do
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
.returns("12345678\n87654321")
run_command("stale_containers").tap do |output|
assert_match /Detected stale container for role web with version 87654321/, output
end
end
test "stop stale_containers" do
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-", raise_on_non_zero_exit: false)
.returns("12345678\n87654321")
run_command("stale_containers", "--stop").tap do |output|
assert_match /Stopping stale container for role web with version 87654321/, output
assert_match /#{Regexp.escape("docker container ls --all --filter name=^app-web-87654321$ --quiet | xargs docker stop")}/, output
end
end
@@ -53,7 +74,7 @@ class CliAppTest < CliTestCase
test "remove" do
run_command("remove").tap do |output|
assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop")}/, output
assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop")}/, output
assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output
assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output
end
@@ -85,7 +106,7 @@ class CliAppTest < CliTestCase
test "exec with reuse" do
run_command("exec", "--reuse", "ruby -v").tap do |output|
assert_match "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", output # Get current version
assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output # Get current version
assert_match "docker exec app-web-999 ruby -v", output
end
end
@@ -104,28 +125,28 @@ class CliAppTest < CliTestCase
test "logs" do
SSHKit::Backend::Abstract.any_instance.stubs(:exec)
.with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 2>&1'")
.with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest| xargs docker logs --timestamps --tail 10 2>&1'")
assert_match "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --tail 100 2>&1", run_command("logs")
assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --tail 100 2>&1", run_command("logs")
end
test "logs with follow" do
SSHKit::Backend::Abstract.any_instance.stubs(:exec)
.with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
.with("ssh -t root@1.1.1.1 'docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1'")
assert_match "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
assert_match "docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow")
end
test "version" do
run_command("version").tap do |output|
assert_match "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", output
assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output
end
end
test "version through main" do
stdouted { Mrsk::Cli::Main.start(["app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1"]) }.tap do |output|
assert_match "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", output
assert_match "docker ps --filter label=service=app --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-", output
end
end

13
test/cli/main_test.rb
View File

@@ -17,6 +17,7 @@ class CliMainTest < CliTestCase
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
@@ -26,6 +27,7 @@ class CliMainTest < CliTestCase
assert_match /Build and push app image/, output
assert_match /Ensure Traefik is running/, output
assert_match /Ensure app can pass healthcheck/, output
assert_match /Detect stale containers/, output
assert_match /Prune old containers and images/, output
end
end
@@ -38,6 +40,7 @@ class CliMainTest < CliTestCase
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
@@ -48,6 +51,7 @@ class CliMainTest < CliTestCase
assert_match /Pull app image/, output
assert_match /Ensure Traefik is running/, output
assert_match /Ensure app can pass healthcheck/, output
assert_match /Detect stale containers/, output
assert_match /Prune old containers and images/, output
assert_match /Releasing the deploy lock/, output
end
@@ -85,6 +89,7 @@ class CliMainTest < CliTestCase
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:server:bootstrap", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options).raises(RuntimeError)
@@ -115,6 +120,7 @@ class CliMainTest < CliTestCase
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
run_command("redeploy").tap do |output|
@@ -128,6 +134,7 @@ class CliMainTest < CliTestCase
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options)
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options)
run_command("redeploy", "--skip_push").tap do |output|
@@ -148,8 +155,8 @@ class CliMainTest < CliTestCase
test "rollback good version" do
Mrsk::Cli::Main.any_instance.stubs(:container_available?).returns(true)
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1").returns("version-to-rollback\n").at_least_once
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=workers", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1").returns("version-to-rollback\n").at_least_once
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-").returns("version-to-rollback\n").at_least_once
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=workers", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-").returns("version-to-rollback\n").at_least_once
run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output|
assert_match "Start version 123", output
@@ -161,7 +168,7 @@ class CliMainTest < CliTestCase
test "rollback without old version" do
Mrsk::Cli::Main.any_instance.stubs(:container_available?).returns(true)
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1").returns("").at_least_once
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-").returns("").at_least_once
run_command("rollback", "123").tap do |output|
assert_match "Start version 123", output

44
test/commands/app_test.rb
View File

@@ -63,14 +63,14 @@ class CommandsAppTest < ActiveSupport::TestCase
test "stop" do
assert_equal \
"docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop",
"docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop",
new_command.stop.join(" ")
end
test "stop with custom stop wait time" do
@config[:stop_wait_time] = 30
assert_equal \
"docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop -t 30",
"docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker stop -t 30",
new_command.stop.join(" ")
end
@@ -96,37 +96,37 @@ class CommandsAppTest < ActiveSupport::TestCase
test "logs" do
assert_equal \
"docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs 2>&1",
"docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs 2>&1",
new_command.logs.join(" ")
assert_equal \
"docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m 2>&1",
"docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --since 5m 2>&1",
new_command.logs(since: "5m").join(" ")
assert_equal \
"docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --tail 100 2>&1",
"docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --tail 100 2>&1",
new_command.logs(lines: "100").join(" ")
assert_equal \
"docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m --tail 100 2>&1",
"docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --since 5m --tail 100 2>&1",
new_command.logs(since: "5m", lines: "100").join(" ")
assert_equal \
"docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs 2>&1 | grep 'my-id'",
"docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs 2>&1 | grep 'my-id'",
new_command.logs(grep: "my-id").join(" ")
assert_equal \
"docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
"docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
new_command.logs(since: "5m", grep: "my-id").join(" ")
end
test "follow logs" do
assert_match \
"docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1",
"docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1",
new_command.follow_logs(host: "app-1")
assert_match \
"docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1 | grep \"Completed\"",
"docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest | xargs docker logs --timestamps --tail 10 --follow 2>&1 | grep \"Completed\"",
new_command.follow_logs(host: "app-1", grep: "Completed")
end
@@ -178,17 +178,17 @@ class CommandsAppTest < ActiveSupport::TestCase
end
test "current_container_id" do
test "current_running_container_id" do
assert_equal \
"docker ps --quiet --filter label=service=app --filter label=role=web",
new_command.current_container_id.join(" ")
"docker ps --quiet --filter label=service=app --filter label=role=web --filter status=running --latest",
new_command.current_running_container_id.join(" ")
end
test "current_container_id with destination" do
test "current_running_container_id with destination" do
@destination = "staging"
assert_equal \
"docker ps --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web",
new_command.current_container_id.join(" ")
"docker ps --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web --filter status=running --latest",
new_command.current_running_container_id.join(" ")
end
test "container_id_for" do
@@ -199,10 +199,20 @@ class CommandsAppTest < ActiveSupport::TestCase
test "current_running_version" do
assert_equal \
"docker ps --filter label=service=app --filter label=role=web --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1",
"docker ps --filter label=service=app --filter label=role=web --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-",
new_command.current_running_version.join(" ")
end
test "list_versions" do
assert_equal \
"docker ps --filter label=service=app --filter label=role=web --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-",
new_command.list_versions.join(" ")
assert_equal \
"docker ps --filter label=service=app --filter label=role=web --filter status=running --latest --format \"{{.Names}}\" | grep -oE \"\\-[^-]+$\" | cut -c 2-",
new_command.list_versions("--latest", status: :running).join(" ")
end
test "list_containers" do
assert_equal \
"docker container ls --all --filter label=service=app --filter label=role=web",