Prune healthcheck containers

If a deployment is interrupted it could leave stale healthcheck
containers around that prevent dependent images from being pruned.
This commit is contained in:
Donal McBreen
2023-08-23 12:03:45 +01:00
parent 9d35793287
commit 718776eb72
7 changed files with 33 additions and 13 deletions

View File

@@ -23,7 +23,8 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
mutating do mutating do
on(KAMAL.hosts) do on(KAMAL.hosts) do
execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug
execute *KAMAL.prune.containers execute *KAMAL.prune.app_containers
execute *KAMAL.prune.healthcheck_containers
end end
end end
end end

View File

@@ -7,8 +7,8 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
"--detach", "--detach",
"--name", container_name_with_version, "--name", container_name_with_version,
"--publish", "#{exposed_port}:#{config.healthcheck["port"]}", "--publish", "#{exposed_port}:#{config.healthcheck["port"]}",
"--label", "service=#{container_name}", "--label", "service=#{config.healthcheck_service}",
"-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"", "-e", "KAMAL_CONTAINER_NAME=\"#{config.healthcheck_service}\"",
*web.env_args, *web.env_args,
*web.health_check_args(cord: false), *web.health_check_args(cord: false),
*config.volume_args, *config.volume_args,
@@ -38,12 +38,8 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
end end
private private
def container_name
[ "healthcheck", config.service, config.destination ].compact.join("-")
end
def container_name_with_version def container_name_with_version
"#{container_name}-#{config.version}" "#{config.healthcheck_service}-#{config.version}"
end end
def container_id def container_id

View File

@@ -13,13 +13,17 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
"while read image tag; do docker rmi $tag; done" "while read image tag; do docker rmi $tag; done"
end end
def containers(keep_last: 5) def app_containers(keep_last: 5)
pipe \ pipe \
docker(:ps, "-q", "-a", *service_filter, *stopped_containers_filters), docker(:ps, "-q", "-a", *service_filter, *stopped_containers_filters),
"tail -n +#{keep_last + 1}", "tail -n +#{keep_last + 1}",
"while read container_id; do docker rm $container_id; done" "while read container_id; do docker rm $container_id; done"
end end
def healthcheck_containers
docker :container, :prune, "--force", *healthcheck_service_filter
end
private private
def stopped_containers_filters def stopped_containers_filters
[ "created", "exited", "dead" ].flat_map { |status| ["--filter", "status=#{status}"] } [ "created", "exited", "dead" ].flat_map { |status| ["--filter", "status=#{status}"] }
@@ -35,4 +39,8 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
def service_filter def service_filter
[ "--filter", "label=service=#{config.service}" ] [ "--filter", "label=service=#{config.service}" ]
end end
end
def healthcheck_service_filter
[ "--filter", "label=service=#{config.healthcheck_service}" ]
end
end

View File

@@ -152,6 +152,10 @@ class Kamal::Configuration
{ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }.merge(raw_config.healthcheck || {}) { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }.merge(raw_config.healthcheck || {})
end end
def healthcheck_service
[ "healthcheck", service, destination ].compact.join("-")
end
def readiness_delay def readiness_delay
raw_config.readiness_delay || 7 raw_config.readiness_delay || 7
end end

View File

@@ -18,6 +18,7 @@ class CliPruneTest < CliTestCase
test "containers" do test "containers" do
run_command("containers").tap do |output| run_command("containers").tap do |output|
assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output
assert_match /docker container prune --force --filter label=service=healthcheck-app on 1.1.1.\d/, output
end end
end end

View File

@@ -20,10 +20,16 @@ class CommandsPruneTest < ActiveSupport::TestCase
new_command.tagged_images.join(" ") new_command.tagged_images.join(" ")
end end
test "containers" do test "app containers" do
assert_equal \ assert_equal \
"docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done", "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done",
new_command.containers.join(" ") new_command.app_containers.join(" ")
end
test "healthcheck containers" do
assert_equal \
"docker container prune --force --filter label=service=healthcheck-app",
new_command.healthcheck_containers.join(" ")
end end
private private

View File

@@ -124,6 +124,10 @@ class ConfigurationTest < ActiveSupport::TestCase
assert_equal "app-missing", @config.service_with_version assert_equal "app-missing", @config.service_with_version
end end
test "healthcheck service" do
assert_equal "healthcheck-app", @config.healthcheck_service
end
test "env with missing secret" do test "env with missing secret" do
assert_raises(KeyError) do assert_raises(KeyError) do
config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!({ config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!({