spacebar/kamal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Minimise holding the deploy lock

Browse Source 
If we get an error we'll only hold the deploy lock if it occurs while
trying to switch the running containers.

We'll also move tagging the latest image from when the image is pulled
to just before the container switch. This ensures that earlier errors
don't leave the hosts with an updated latest tag while still running the
older version.
This commit is contained in:
Donal McBreen
2023-04-12 09:37:20 +01:00
parent 60a19f0b30
commit 051556674f
10 changed files with 124 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
lib/mrsk/cli/app.rb
View File

@@ -8,25 +8,28 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
cli = self
on(MRSK.hosts) do
execute *MRSK.auditor.record("Tagging #{MRSK.config.absolute_image} as the latest image"), verbosity: :debug
execute *MRSK.app.tag_current_as_latest
end
on(MRSK.hosts) do |host|
roles = MRSK.roles_on(host)
roles.each do |role|
execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug
begin
if capture_with_info(*MRSK.app(role: role).container_id_for_version(version)).present?
tmp_version = "#{version}_#{SecureRandom.hex(8)}"
info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
execute *MRSK.auditor(role: role).record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
execute *MRSK.app(role: role).rename_container(version: version, new_version: tmp_version)
end
old_version = capture_with_info(*MRSK.app(role: role).current_running_version).strip
execute *MRSK.app(role: role).run
sleep MRSK.config.readiness_delay
execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
if capture_with_info(*MRSK.app(role: role).container_id_for_version(version)).present?
tmp_version = "#{version}_#{SecureRandom.hex(8)}"
info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
execute *MRSK.auditor(role: role).record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
execute *MRSK.app(role: role).rename_container(version: version, new_version: tmp_version)
end
old_version = capture_with_info(*MRSK.app(role: role).current_running_version).strip
execute *MRSK.app(role: role).run
sleep MRSK.config.readiness_delay
execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
end
end
end

49
lib/mrsk/cli/base.rb
View File

@@ -77,22 +77,32 @@ module Mrsk::Cli
end
def with_lock
acquire_lock
if MRSK.holding_lock?
yield
else
acquire_lock
yield
begin
yield
rescue
if MRSK.hold_lock_on_error?
error " \e[31mDeploy lock was not released\e[0m"
else
release_lock
end
release_lock
rescue
error " \e[31mDeploy lock was not released\e[0m" if MRSK.lock_count > 0
raise
raise
end
release_lock
end
end
def acquire_lock
if MRSK.lock_count == 0
say "Acquiring the deploy lock"
on(MRSK.primary_host) { execute *MRSK.lock.acquire("Automatic deploy lock", MRSK.config.version) }
end
MRSK.lock_count += 1
say "Acquiring the deploy lock"
on(MRSK.primary_host) { execute *MRSK.lock.acquire("Automatic deploy lock", MRSK.config.version) }
MRSK.holding_lock = true
rescue SSHKit::Runner::ExecuteError => e
if e.message =~ /cannot create directory/
invoke "mrsk:cli:lock:status", []
@@ -103,10 +113,19 @@ module Mrsk::Cli
end
def release_lock
MRSK.lock_count -= 1
if MRSK.lock_count == 0
say "Releasing the deploy lock"
on(MRSK.primary_host) { execute *MRSK.lock.release }
say "Releasing the deploy lock"
on(MRSK.primary_host) { execute *MRSK.lock.release }
MRSK.holding_lock = false
end
def hold_lock_on_error
if MRSK.hold_lock_on_error?
yield
else
MRSK.hold_lock_on_error = true
yield
MRSK.hold_lock_on_error = false
end
end
end

51
lib/mrsk/cli/main.rb
View File

@@ -37,7 +37,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
say "Ensure app can pass healthcheck...", :magenta
invoke "mrsk:cli:healthcheck:perform", [], invoke_options
invoke "mrsk:cli:app:boot", [], invoke_options
hold_lock_on_error do
invoke "mrsk:cli:app:boot", [], invoke_options
end
say "Prune old containers and images...", :magenta
invoke "mrsk:cli:prune:all", [], invoke_options
@@ -65,7 +67,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
say "Ensure app can pass healthcheck...", :magenta
invoke "mrsk:cli:healthcheck:perform", [], invoke_options
invoke "mrsk:cli:app:boot", [], invoke_options
hold_lock_on_error do
invoke "mrsk:cli:app:boot", [], invoke_options
end
end
audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
@@ -75,34 +79,41 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
desc "rollback [VERSION]", "Rollback app to VERSION"
def rollback(version)
with_lock do
MRSK.config.version = version
invoke_options = deploy_options
if container_available?(version)
say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta
cli = self
hold_lock_on_error do
MRSK.config.version = version
old_version = nil
on(MRSK.hosts) do |host|
roles = MRSK.roles_on(host)
if container_available?(version)
say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta
roles.each do |role|
app = MRSK.app(role: role)
old_version = capture_with_info(*app.current_running_version).strip.presence
on(MRSK.hosts) do
execute *MRSK.auditor.record("Tagging #{MRSK.config.absolute_image} as the latest image"), verbosity: :debug
execute *MRSK.app.tag_current_as_latest
end
execute *app.start
on(MRSK.hosts) do |host|
roles = MRSK.roles_on(host)
if old_version
sleep MRSK.config.readiness_delay
roles.each do |role|
app = MRSK.app(role: role)
old_version = capture_with_info(*app.current_running_version).strip.presence
execute *app.stop(version: old_version), raise_on_non_zero_exit: false
execute *app.start
if old_version
sleep MRSK.config.readiness_delay
execute *app.stop(version: old_version), raise_on_non_zero_exit: false
end
end
end
end
audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast]
else
say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast]
else
say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
end
end
end
end

13
lib/mrsk/commander.rb
View File

@@ -2,11 +2,12 @@ require "active_support/core_ext/enumerable"
require "active_support/core_ext/module/delegation"
class Mrsk::Commander
attr_accessor :verbosity, :lock_count
attr_accessor :verbosity, :holding_lock, :hold_lock_on_error
def initialize
self.verbosity = :info
self.lock_count = 0
self.holding_lock = false
self.hold_lock_on_error = false
end
def config
@@ -115,6 +116,14 @@ class Mrsk::Commander
SSHKit.config.output_verbosity = old_level
end
def holding_lock?
self.holding_lock
end
def hold_lock_on_error?
self.hold_lock_on_error
end
private
# Lazy setup of SSHKit
def configure_sshkit_with(config)

4
lib/mrsk/commands/app.rb
View File

@@ -128,6 +128,10 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
docker :image, :prune, "--all", "--force", *filter_args
end
def tag_current_as_latest
docker :tag, config.absolute_image, config.latest_image
end
private
def container_name(version = nil)

1
lib/mrsk/commands/builder/base.rb
View File

@@ -7,7 +7,6 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
def pull
docker :pull, config.absolute_image
docker :pull, config.latest_image
end
def build_options