Healthcheck polling is a CLI concern
Also, it has no instance variables, so let's just have it be a module.
This commit is contained in:
@@ -44,14 +44,14 @@ class Kamal::Cli::App < Kamal::Cli::Base
|
|||||||
|
|
||||||
execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
|
execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
|
||||||
|
|
||||||
Kamal::Utils::HealthcheckPoller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
|
Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
|
||||||
|
|
||||||
if old_version.present?
|
if old_version.present?
|
||||||
if role_config.uses_cord?
|
if role_config.uses_cord?
|
||||||
cord = capture_with_info(*app.cord(version: old_version), raise_on_non_zero_exit: false).strip
|
cord = capture_with_info(*app.cord(version: old_version), raise_on_non_zero_exit: false).strip
|
||||||
if cord.present?
|
if cord.present?
|
||||||
execute *app.cut_cord(cord)
|
execute *app.cut_cord(cord)
|
||||||
Kamal::Utils::HealthcheckPoller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
|
Kamal::Cli::Healthcheck::Poller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ class Kamal::Cli::Healthcheck < Kamal::Cli::Base
|
|||||||
on(KAMAL.primary_host) do
|
on(KAMAL.primary_host) do
|
||||||
begin
|
begin
|
||||||
execute *KAMAL.healthcheck.run
|
execute *KAMAL.healthcheck.run
|
||||||
Kamal::Utils::HealthcheckPoller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
|
Poller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
|
||||||
rescue Kamal::Utils::HealthcheckPoller::HealthcheckError => e
|
rescue Poller::HealthcheckError => e
|
||||||
error capture_with_info(*KAMAL.healthcheck.logs)
|
error capture_with_info(*KAMAL.healthcheck.logs)
|
||||||
error capture_with_pretty_json(*KAMAL.healthcheck.container_health_log)
|
error capture_with_pretty_json(*KAMAL.healthcheck.container_health_log)
|
||||||
raise
|
raise
|
||||||
|
|||||||
64
lib/kamal/cli/healthcheck/poller.rb
Normal file
64
lib/kamal/cli/healthcheck/poller.rb
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
module Kamal::Cli::Healthcheck::Poller
|
||||||
|
extend self
|
||||||
|
|
||||||
|
TRAEFIK_UPDATE_DELAY = 5
|
||||||
|
|
||||||
|
class HealthcheckError < StandardError; end
|
||||||
|
|
||||||
|
def wait_for_healthy(pause_after_ready: false, &block)
|
||||||
|
attempt = 1
|
||||||
|
max_attempts = KAMAL.config.healthcheck["max_attempts"]
|
||||||
|
|
||||||
|
begin
|
||||||
|
case status = block.call
|
||||||
|
when "healthy"
|
||||||
|
sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
|
||||||
|
when "running" # No health check configured
|
||||||
|
sleep KAMAL.config.readiness_delay if pause_after_ready
|
||||||
|
else
|
||||||
|
raise HealthcheckError, "container not ready (#{status})"
|
||||||
|
end
|
||||||
|
rescue HealthcheckError => e
|
||||||
|
if attempt <= max_attempts
|
||||||
|
info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
|
||||||
|
sleep attempt
|
||||||
|
attempt += 1
|
||||||
|
retry
|
||||||
|
else
|
||||||
|
raise
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
info "Container is healthy!"
|
||||||
|
end
|
||||||
|
|
||||||
|
def wait_for_unhealthy(pause_after_ready: false, &block)
|
||||||
|
attempt = 1
|
||||||
|
max_attempts = KAMAL.config.healthcheck["max_attempts"]
|
||||||
|
|
||||||
|
begin
|
||||||
|
case status = block.call
|
||||||
|
when "unhealthy"
|
||||||
|
sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
|
||||||
|
else
|
||||||
|
raise HealthcheckError, "container not unhealthy (#{status})"
|
||||||
|
end
|
||||||
|
rescue HealthcheckError => e
|
||||||
|
if attempt <= max_attempts
|
||||||
|
info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
|
||||||
|
sleep attempt
|
||||||
|
attempt += 1
|
||||||
|
retry
|
||||||
|
else
|
||||||
|
raise
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
info "Container is unhealthy!"
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
def info(message)
|
||||||
|
SSHKit.config.output.info(message)
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -1,64 +0,0 @@
|
|||||||
class Kamal::Utils::HealthcheckPoller
|
|
||||||
TRAEFIK_UPDATE_DELAY = 5
|
|
||||||
|
|
||||||
class HealthcheckError < StandardError; end
|
|
||||||
|
|
||||||
class << self
|
|
||||||
def wait_for_healthy(pause_after_ready: false, &block)
|
|
||||||
attempt = 1
|
|
||||||
max_attempts = KAMAL.config.healthcheck["max_attempts"]
|
|
||||||
|
|
||||||
begin
|
|
||||||
case status = block.call
|
|
||||||
when "healthy"
|
|
||||||
sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
|
|
||||||
when "running" # No health check configured
|
|
||||||
sleep KAMAL.config.readiness_delay if pause_after_ready
|
|
||||||
else
|
|
||||||
raise HealthcheckError, "container not ready (#{status})"
|
|
||||||
end
|
|
||||||
rescue HealthcheckError => e
|
|
||||||
if attempt <= max_attempts
|
|
||||||
info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
|
|
||||||
sleep attempt
|
|
||||||
attempt += 1
|
|
||||||
retry
|
|
||||||
else
|
|
||||||
raise
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
info "Container is healthy!"
|
|
||||||
end
|
|
||||||
|
|
||||||
def wait_for_unhealthy(pause_after_ready: false, &block)
|
|
||||||
attempt = 1
|
|
||||||
max_attempts = KAMAL.config.healthcheck["max_attempts"]
|
|
||||||
|
|
||||||
begin
|
|
||||||
case status = block.call
|
|
||||||
when "unhealthy"
|
|
||||||
sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
|
|
||||||
else
|
|
||||||
raise HealthcheckError, "container not unhealthy (#{status})"
|
|
||||||
end
|
|
||||||
rescue HealthcheckError => e
|
|
||||||
if attempt <= max_attempts
|
|
||||||
info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
|
|
||||||
sleep attempt
|
|
||||||
attempt += 1
|
|
||||||
retry
|
|
||||||
else
|
|
||||||
raise
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
info "Container is unhealthy!"
|
|
||||||
end
|
|
||||||
|
|
||||||
private
|
|
||||||
def info(message)
|
|
||||||
SSHKit.config.output.info(message)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
@@ -5,7 +5,7 @@ class CliHealthcheckTest < CliTestCase
|
|||||||
# Prevent expected failures from outputting to terminal
|
# Prevent expected failures from outputting to terminal
|
||||||
Thread.report_on_exception = false
|
Thread.report_on_exception = false
|
||||||
|
|
||||||
Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
|
Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
|
||||||
Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
|
Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
|
||||||
|
|
||||||
SSHKit::Backend::Abstract.any_instance.stubs(:execute)
|
SSHKit::Backend::Abstract.any_instance.stubs(:execute)
|
||||||
@@ -35,7 +35,7 @@ class CliHealthcheckTest < CliTestCase
|
|||||||
# Prevent expected failures from outputting to terminal
|
# Prevent expected failures from outputting to terminal
|
||||||
Thread.report_on_exception = false
|
Thread.report_on_exception = false
|
||||||
|
|
||||||
Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
|
Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
|
||||||
|
|
||||||
SSHKit::Backend::Abstract.any_instance.stubs(:execute)
|
SSHKit::Backend::Abstract.any_instance.stubs(:execute)
|
||||||
.with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
|
.with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)
|
||||||
|
|||||||
@@ -216,7 +216,7 @@ class CliMainTest < CliTestCase
|
|||||||
test "rollback without old version" do
|
test "rollback without old version" do
|
||||||
Kamal::Cli::Main.any_instance.stubs(:container_available?).returns(true)
|
Kamal::Cli::Main.any_instance.stubs(:container_available?).returns(true)
|
||||||
|
|
||||||
Kamal::Utils::HealthcheckPoller.stubs(:sleep)
|
Kamal::Cli::Healthcheck::Poller.stubs(:sleep)
|
||||||
|
|
||||||
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
|
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
|
||||||
.with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
|
.with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
|
||||||
|
|||||||
Reference in New Issue
Block a user