Show container logs and healthcheck status on failure

This commit is contained in:
Donal McBreen
2024-05-07 17:07:21 +01:00
parent 5be6fa3b4e
commit 773ba3a5ab
7 changed files with 31 additions and 14 deletions

View File

@@ -15,7 +15,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
end end
#  Primary hosts and roles are returned first, so they can open the barrier #  Primary hosts and roles are returned first, so they can open the barrier
barrier = Kamal::Cli::Healthcheck::Barrier.new if KAMAL.roles.many? barrier = Kamal::Cli::Healthcheck::Barrier.new
on(KAMAL.hosts, **KAMAL.boot_strategy) do |host| on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
KAMAL.roles_on(host).each do |role| KAMAL.roles_on(host).each do |role|

View File

@@ -1,6 +1,6 @@
class Kamal::Cli::App::Boot class Kamal::Cli::App::Boot
attr_reader :host, :role, :version, :barrier, :sshkit attr_reader :host, :role, :version, :barrier, :sshkit
delegate :execute, :capture_with_info, :info, to: :sshkit delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, to: :sshkit
delegate :uses_cord?, :assets?, :running_traefik?, to: :role delegate :uses_cord?, :assets?, :running_traefik?, to: :role
def initialize(host, role, sshkit, version, barrier) def initialize(host, role, sshkit, version, barrier)
@@ -55,7 +55,11 @@ class Kamal::Cli::App::Boot
reach_barrier reach_barrier
rescue => e rescue => e
close_barrier if barrier_role? if barrier_role? && barrier.close
info "Deploy failed, so closed barrier (#{host})"
error capture_with_info(*app.logs(version: version))
error capture_with_info(*app.container_health_log(version: version))
end
execute *app.stop(version: version), raise_on_non_zero_exit: false execute *app.stop(version: version), raise_on_non_zero_exit: false
raise raise
@@ -92,14 +96,10 @@ class Kamal::Cli::App::Boot
barrier.wait barrier.wait
info "Barrier opened (#{host})" info "Barrier opened (#{host})"
rescue Kamal::Cli::Healthcheck::Error rescue Kamal::Cli::Healthcheck::Error
info "Barrier closed, shutting down new container... (#{host})" info "Barrier closed, shutting down new container (#{host})..."
raise raise
end end
def close_barrier
barrier&.close
end
def barrier_role? def barrier_role?
role == KAMAL.primary_role role == KAMAL.primary_role
end end

View File

@@ -1,4 +1,6 @@
module Kamal::Commands::App::Containers module Kamal::Commands::App::Containers
DOCKER_HEALTH_LOG_FORMAT = "'{{json .State.Health}}'"
def list_containers def list_containers
docker :container, :ls, "--all", *filter_args docker :container, :ls, "--all", *filter_args
end end
@@ -20,4 +22,10 @@ module Kamal::Commands::App::Containers
def remove_containers def remove_containers
docker :container, :prune, "--force", *filter_args docker :container, :prune, "--force", *filter_args
end end
def container_health_log(version:)
pipe \
container_id_for(container_name: container_name(version)),
xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT))
end
end end

View File

@@ -1,7 +1,7 @@
module Kamal::Commands::App::Logging module Kamal::Commands::App::Logging
def logs(since: nil, lines: nil, grep: nil) def logs(version: nil, since: nil, lines: nil, grep: nil)
pipe \ pipe \
current_running_container_id, version ? container_id_for_version(version) : current_running_container_id,
"xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1", "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
("grep '#{grep}'" if grep) ("grep '#{grep}'" if grep)
end end

View File

@@ -3,7 +3,6 @@ module Kamal::Commands
delegate :sensitive, :argumentize, to: Kamal::Utils delegate :sensitive, :argumentize, to: Kamal::Utils
DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'" DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'"
DOCKER_HEALTH_LOG_FORMAT = "'{{json .State.Health}}'"
attr_accessor :config attr_accessor :config

View File

@@ -162,8 +162,8 @@ class CliAppTest < CliTestCase
run_command("boot", config: :with_roles, host: nil, allow_execute_error: true).tap do |output| run_command("boot", config: :with_roles, host: nil, allow_execute_error: true).tap do |output|
assert_match "Waiting at web barrier (1.1.1.3)...", output assert_match "Waiting at web barrier (1.1.1.3)...", output
assert_match "Waiting at web barrier (1.1.1.4)...", output assert_match "Waiting at web barrier (1.1.1.4)...", output
assert_match "Barrier closed, shutting down new container... (1.1.1.3)", output assert_match "Barrier closed, shutting down new container (1.1.1.3)...", output
assert_match "Barrier closed, shutting down new container... (1.1.1.4)", output assert_match "Barrier closed, shutting down new container (1.1.1.4)...", output
assert_match "Running docker container ls --all --filter name=^app-web-latest$ --quiet | xargs docker stop on 1.1.1.1", output assert_match "Running docker container ls --all --filter name=^app-web-latest$ --quiet | xargs docker stop on 1.1.1.1", output
assert_match "Running docker container ls --all --filter name=^app-web-latest$ --quiet | xargs docker stop on 1.1.1.2", output assert_match "Running docker container ls --all --filter name=^app-web-latest$ --quiet | xargs docker stop on 1.1.1.2", output
assert_match "Running docker container ls --all --filter name=^app-workers-latest$ --quiet | xargs docker stop on 1.1.1.3", output assert_match "Running docker container ls --all --filter name=^app-workers-latest$ --quiet | xargs docker stop on 1.1.1.3", output

View File

@@ -15,10 +15,20 @@ class BrokenDeployTest < IntegrationTest
second_version = break_app second_version = break_app
kamal :deploy, raise_on_error: false output = kamal :deploy, raise_on_error: false, capture: true
assert_failed_deploy output
assert_app_is_up version: first_version assert_app_is_up version: first_version
assert_container_running host: :vm3, name: "app-workers-#{first_version}" assert_container_running host: :vm3, name: "app-workers-#{first_version}"
assert_container_not_running host: :vm3, name: "app-workers-#{second_version}" assert_container_not_running host: :vm3, name: "app-workers-#{second_version}"
end end
private
def assert_failed_deploy(output)
assert_match "Waiting at web barrier (vm3)...", output
assert_match /Deploy failed, so closed barrier \(vm[12]\)/, output
assert_match "Barrier closed, shutting down new container (vm3)...", output
assert_match "nginx: [emerg] unexpected end of file, expecting \";\" or \"}\" in /etc/nginx/conf.d/default.conf:2", output
assert_match 'ERROR {"Status":"unhealthy","FailingStreak":0,"Log":[]}', output
end
end end