From 1ccf679ca9eea14ea91413a532c96cba1993cbf1 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Mon, 20 Feb 2023 17:42:55 +0100 Subject: [PATCH 1/5] Switch to ruby-based retry Retry connection errors with backoff --- lib/mrsk/cli/healthcheck.rb | 27 +++++++++++++++++++++------ lib/mrsk/commands/healthcheck.rb | 4 ++-- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/lib/mrsk/cli/healthcheck.rb b/lib/mrsk/cli/healthcheck.rb index a9fcf803..24e09ce6 100644 --- a/lib/mrsk/cli/healthcheck.rb +++ b/lib/mrsk/cli/healthcheck.rb @@ -1,4 +1,6 @@ class Mrsk::Cli::Healthcheck < Mrsk::Cli::Base + MAX_ATTEMPTS = 5 + default_command :perform desc "perform", "Health check current app version" @@ -8,18 +10,31 @@ class Mrsk::Cli::Healthcheck < Mrsk::Cli::Base execute *MRSK.healthcheck.run target = "Health check against #{MRSK.config.healthcheck["path"]}" + attempt = 1 - if capture_with_info(*MRSK.healthcheck.curl) == "200" - info "#{target} succeeded with 200 OK!" - else - # Catches 1xx, 2xx, 3xx - raise SSHKit::Command::Failed, "#{target} failed to return 200 OK!" + begin + status = capture_with_info(*MRSK.healthcheck.curl) + + if status == "200" + info "#{target} succeeded with 200 OK!" + else + raise "#{target} failed with status #{status}" + end + rescue SSHKit::Command::Failed + if attempt <= MAX_ATTEMPTS + info "#{target} not ready yet, retrying in #{attempt}s..." + sleep attempt + attempt += 1 + + retry + else + raise + end end rescue SSHKit::Command::Failed => e error capture_with_info(*MRSK.healthcheck.logs) if e.message =~ /curl/ - # Catches 4xx, 5xx raise SSHKit::Command::Failed, "#{target} failed to return 200 OK!" else raise diff --git a/lib/mrsk/commands/healthcheck.rb b/lib/mrsk/commands/healthcheck.rb index feb35543..47626156 100644 --- a/lib/mrsk/commands/healthcheck.rb +++ b/lib/mrsk/commands/healthcheck.rb @@ -16,11 +16,11 @@ class Mrsk::Commands::Healthcheck < Mrsk::Commands::Base end def curl - [ :curl, "--silent", "--output", "/dev/null", "--write-out", "'%{http_code}'", "--retry-max-time", 10, "--retry", 5, health_url ] + [ :curl, "--silent", "--output", "/dev/null", "--write-out", "'%{http_code}'", "--max-time", "2", health_url ] end def logs - pipe container_id, xargs(docker(:logs, "2>&1")) + pipe container_id, xargs(docker(:logs, "--tail", 50, "2>&1")) end def stop From c4cc9e690bfa7b302c3675afbf48f1969a60f4bc Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Mon, 20 Feb 2023 17:44:55 +0100 Subject: [PATCH 2/5] Better language --- lib/mrsk/cli/healthcheck.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mrsk/cli/healthcheck.rb b/lib/mrsk/cli/healthcheck.rb index 24e09ce6..ffb0805d 100644 --- a/lib/mrsk/cli/healthcheck.rb +++ b/lib/mrsk/cli/healthcheck.rb @@ -22,7 +22,7 @@ class Mrsk::Cli::Healthcheck < Mrsk::Cli::Base end rescue SSHKit::Command::Failed if attempt <= MAX_ATTEMPTS - info "#{target} not ready yet, retrying in #{attempt}s..." + info "#{target} failed to respond, retrying in #{attempt}s..." sleep attempt attempt += 1 From 9d6ccf988993418858d5337c8d8145cda3769cf5 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Mon, 20 Feb 2023 17:59:41 +0100 Subject: [PATCH 3/5] Changed to a timeout --- test/commands/healthcheck_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/commands/healthcheck_test.rb b/test/commands/healthcheck_test.rb index c7438126..54580e61 100644 --- a/test/commands/healthcheck_test.rb +++ b/test/commands/healthcheck_test.rb @@ -24,7 +24,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase test "curl" do assert_equal \ - "curl --silent --output /dev/null --write-out '%{http_code}' --retry-max-time 10 --retry 5 http://localhost:3999/up", + "curl --silent --output /dev/null --write-out '%{http_code}' --max-time 2 http://localhost:3999/up", new_command.curl.join(" ") end @@ -32,7 +32,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase @config[:healthcheck] = { "path" => "/healthz" } assert_equal \ - "curl --silent --output /dev/null --write-out '%{http_code}' --retry-max-time 10 --retry 5 http://localhost:3999/healthz", + "curl --silent --output /dev/null --write-out '%{http_code}' --max-time 2 http://localhost:3999/healthz", new_command.curl.join(" ") end From 42b343436d3c42a1d23116582ec577b333a814da Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Mon, 20 Feb 2023 18:06:16 +0100 Subject: [PATCH 4/5] Remove images of the same name before pulling a new one Or you'll end up with untagged dupes. --- lib/mrsk/cli/build.rb | 1 + lib/mrsk/commands/builder.rb | 2 +- lib/mrsk/commands/builder/base.rb | 4 ++++ test/cli/build_test.rb | 15 +++++++++++++++ 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 test/cli/build_test.rb diff --git a/lib/mrsk/cli/build.rb b/lib/mrsk/cli/build.rb index 5e493a56..3e8912d7 100644 --- a/lib/mrsk/cli/build.rb +++ b/lib/mrsk/cli/build.rb @@ -30,6 +30,7 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base def pull on(MRSK.hosts) do execute *MRSK.auditor.record("Pulled image with version #{MRSK.version}"), verbosity: :debug + execute *MRSK.builder.clean, raise_on_non_zero_exit: false execute *MRSK.builder.pull end end diff --git a/lib/mrsk/commands/builder.rb b/lib/mrsk/commands/builder.rb index 70957afe..85dbce0a 100644 --- a/lib/mrsk/commands/builder.rb +++ b/lib/mrsk/commands/builder.rb @@ -1,5 +1,5 @@ class Mrsk::Commands::Builder < Mrsk::Commands::Base - delegate :create, :remove, :push, :pull, :info, to: :target + delegate :create, :remove, :push, :clean, :pull, :info, to: :target def name target.class.to_s.remove("Mrsk::Commands::Builder::").underscore diff --git a/lib/mrsk/commands/builder/base.rb b/lib/mrsk/commands/builder/base.rb index 4f2a8dbe..d282fd01 100644 --- a/lib/mrsk/commands/builder/base.rb +++ b/lib/mrsk/commands/builder/base.rb @@ -1,6 +1,10 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base delegate :argumentize, to: Mrsk::Utils + def clean + docker :image, :rm, "--force", config.absolute_image + end + def pull docker :pull, config.absolute_image end diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb new file mode 100644 index 00000000..3d5f4431 --- /dev/null +++ b/test/cli/build_test.rb @@ -0,0 +1,15 @@ +require_relative "cli_test_case" + +class CliBuildTest < CliTestCase + test "pull" do + run_command("pull").tap do |output| + assert_match /docker image rm --force dhh\/app:999/, output + assert_match /docker pull dhh\/app:999/, output + end + end + + private + def run_command(*command) + stdouted { Mrsk::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) } + end +end From f6ea287e66d9b42890be07a7ee53226ee996da85 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Mon, 20 Feb 2023 18:06:56 +0100 Subject: [PATCH 5/5] Bump version for 0.8.0 --- Gemfile.lock | 2 +- lib/mrsk/version.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 772ef88f..b0cbf611 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - mrsk (0.7.2) + mrsk (0.8.0) activesupport (>= 7.0) dotenv (~> 2.8) sshkit (~> 1.21) diff --git a/lib/mrsk/version.rb b/lib/mrsk/version.rb index ecdce2cb..11d95f38 100644 --- a/lib/mrsk/version.rb +++ b/lib/mrsk/version.rb @@ -1,3 +1,3 @@ module Mrsk - VERSION = "0.7.2" + VERSION = "0.8.0" end