Show container logs and healthcheck status on failure
This commit is contained in:
@@ -15,7 +15,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
|
|||||||
end
|
end
|
||||||
|
|
||||||
# Primary hosts and roles are returned first, so they can open the barrier
|
# Primary hosts and roles are returned first, so they can open the barrier
|
||||||
barrier = Kamal::Cli::Healthcheck::Barrier.new if KAMAL.roles.many?
|
barrier = Kamal::Cli::Healthcheck::Barrier.new
|
||||||
|
|
||||||
on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
|
on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
|
||||||
KAMAL.roles_on(host).each do |role|
|
KAMAL.roles_on(host).each do |role|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
class Kamal::Cli::App::Boot
|
class Kamal::Cli::App::Boot
|
||||||
attr_reader :host, :role, :version, :barrier, :sshkit
|
attr_reader :host, :role, :version, :barrier, :sshkit
|
||||||
delegate :execute, :capture_with_info, :info, to: :sshkit
|
delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, to: :sshkit
|
||||||
delegate :uses_cord?, :assets?, :running_traefik?, to: :role
|
delegate :uses_cord?, :assets?, :running_traefik?, to: :role
|
||||||
|
|
||||||
def initialize(host, role, sshkit, version, barrier)
|
def initialize(host, role, sshkit, version, barrier)
|
||||||
@@ -55,7 +55,11 @@ class Kamal::Cli::App::Boot
|
|||||||
|
|
||||||
reach_barrier
|
reach_barrier
|
||||||
rescue => e
|
rescue => e
|
||||||
close_barrier if barrier_role?
|
if barrier_role? && barrier.close
|
||||||
|
info "Deploy failed, so closed barrier (#{host})"
|
||||||
|
error capture_with_info(*app.logs(version: version))
|
||||||
|
error capture_with_info(*app.container_health_log(version: version))
|
||||||
|
end
|
||||||
execute *app.stop(version: version), raise_on_non_zero_exit: false
|
execute *app.stop(version: version), raise_on_non_zero_exit: false
|
||||||
|
|
||||||
raise
|
raise
|
||||||
@@ -92,14 +96,10 @@ class Kamal::Cli::App::Boot
|
|||||||
barrier.wait
|
barrier.wait
|
||||||
info "Barrier opened (#{host})"
|
info "Barrier opened (#{host})"
|
||||||
rescue Kamal::Cli::Healthcheck::Error
|
rescue Kamal::Cli::Healthcheck::Error
|
||||||
info "Barrier closed, shutting down new container... (#{host})"
|
info "Barrier closed, shutting down new container (#{host})..."
|
||||||
raise
|
raise
|
||||||
end
|
end
|
||||||
|
|
||||||
def close_barrier
|
|
||||||
barrier&.close
|
|
||||||
end
|
|
||||||
|
|
||||||
def barrier_role?
|
def barrier_role?
|
||||||
role == KAMAL.primary_role
|
role == KAMAL.primary_role
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
module Kamal::Commands::App::Containers
|
module Kamal::Commands::App::Containers
|
||||||
|
DOCKER_HEALTH_LOG_FORMAT = "'{{json .State.Health}}'"
|
||||||
|
|
||||||
def list_containers
|
def list_containers
|
||||||
docker :container, :ls, "--all", *filter_args
|
docker :container, :ls, "--all", *filter_args
|
||||||
end
|
end
|
||||||
@@ -20,4 +22,10 @@ module Kamal::Commands::App::Containers
|
|||||||
def remove_containers
|
def remove_containers
|
||||||
docker :container, :prune, "--force", *filter_args
|
docker :container, :prune, "--force", *filter_args
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def container_health_log(version:)
|
||||||
|
pipe \
|
||||||
|
container_id_for(container_name: container_name(version)),
|
||||||
|
xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT))
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
module Kamal::Commands::App::Logging
|
module Kamal::Commands::App::Logging
|
||||||
def logs(since: nil, lines: nil, grep: nil)
|
def logs(version: nil, since: nil, lines: nil, grep: nil)
|
||||||
pipe \
|
pipe \
|
||||||
current_running_container_id,
|
version ? container_id_for_version(version) : current_running_container_id,
|
||||||
"xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
|
"xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
|
||||||
("grep '#{grep}'" if grep)
|
("grep '#{grep}'" if grep)
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ module Kamal::Commands
|
|||||||
delegate :sensitive, :argumentize, to: Kamal::Utils
|
delegate :sensitive, :argumentize, to: Kamal::Utils
|
||||||
|
|
||||||
DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'"
|
DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'"
|
||||||
DOCKER_HEALTH_LOG_FORMAT = "'{{json .State.Health}}'"
|
|
||||||
|
|
||||||
attr_accessor :config
|
attr_accessor :config
|
||||||
|
|
||||||
|
|||||||
@@ -162,8 +162,8 @@ class CliAppTest < CliTestCase
|
|||||||
run_command("boot", config: :with_roles, host: nil, allow_execute_error: true).tap do |output|
|
run_command("boot", config: :with_roles, host: nil, allow_execute_error: true).tap do |output|
|
||||||
assert_match "Waiting at web barrier (1.1.1.3)...", output
|
assert_match "Waiting at web barrier (1.1.1.3)...", output
|
||||||
assert_match "Waiting at web barrier (1.1.1.4)...", output
|
assert_match "Waiting at web barrier (1.1.1.4)...", output
|
||||||
assert_match "Barrier closed, shutting down new container... (1.1.1.3)", output
|
assert_match "Barrier closed, shutting down new container (1.1.1.3)...", output
|
||||||
assert_match "Barrier closed, shutting down new container... (1.1.1.4)", output
|
assert_match "Barrier closed, shutting down new container (1.1.1.4)...", output
|
||||||
assert_match "Running docker container ls --all --filter name=^app-web-latest$ --quiet | xargs docker stop on 1.1.1.1", output
|
assert_match "Running docker container ls --all --filter name=^app-web-latest$ --quiet | xargs docker stop on 1.1.1.1", output
|
||||||
assert_match "Running docker container ls --all --filter name=^app-web-latest$ --quiet | xargs docker stop on 1.1.1.2", output
|
assert_match "Running docker container ls --all --filter name=^app-web-latest$ --quiet | xargs docker stop on 1.1.1.2", output
|
||||||
assert_match "Running docker container ls --all --filter name=^app-workers-latest$ --quiet | xargs docker stop on 1.1.1.3", output
|
assert_match "Running docker container ls --all --filter name=^app-workers-latest$ --quiet | xargs docker stop on 1.1.1.3", output
|
||||||
|
|||||||
@@ -15,10 +15,20 @@ class BrokenDeployTest < IntegrationTest
|
|||||||
|
|
||||||
second_version = break_app
|
second_version = break_app
|
||||||
|
|
||||||
kamal :deploy, raise_on_error: false
|
output = kamal :deploy, raise_on_error: false, capture: true
|
||||||
|
|
||||||
|
assert_failed_deploy output
|
||||||
assert_app_is_up version: first_version
|
assert_app_is_up version: first_version
|
||||||
assert_container_running host: :vm3, name: "app-workers-#{first_version}"
|
assert_container_running host: :vm3, name: "app-workers-#{first_version}"
|
||||||
assert_container_not_running host: :vm3, name: "app-workers-#{second_version}"
|
assert_container_not_running host: :vm3, name: "app-workers-#{second_version}"
|
||||||
end
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
def assert_failed_deploy(output)
|
||||||
|
assert_match "Waiting at web barrier (vm3)...", output
|
||||||
|
assert_match /Deploy failed, so closed barrier \(vm[12]\)/, output
|
||||||
|
assert_match "Barrier closed, shutting down new container (vm3)...", output
|
||||||
|
assert_match "nginx: [emerg] unexpected end of file, expecting \";\" or \"}\" in /etc/nginx/conf.d/default.conf:2", output
|
||||||
|
assert_match 'ERROR {"Status":"unhealthy","FailingStreak":0,"Log":[]}', output
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
Reference in New Issue
Block a user