Zero downtime redeploys

When deploying check if there is already a container with the existing
name. If there is rename it to "<version>_<random_hex_string>" to remove
the name clash with the new container we want to boot.

We can then do the normal zero downtime run/wait/stop.

While implementing this I discovered the --filter name=foo does a
substring match for foo, so I've updated those filters to do an exact
match instead.
This commit is contained in:
Donal McBreen
2023-03-24 17:06:54 +00:00
parent 01a2b678d7
commit 05488e4c1e
12 changed files with 48 additions and 48 deletions

View File

@@ -15,22 +15,17 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug
begin
if capture_with_info(*MRSK.app(role: role).container_id_for_version(version)).present?
tmp_version = "#{version}_#{SecureRandom.hex(8)}"
info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
execute *MRSK.auditor(role: role).record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
execute *MRSK.app(role: role).rename_container(version: version, new_version: tmp_version)
end
old_version = capture_with_info(*MRSK.app(role: role).current_running_version).strip
execute *MRSK.app(role: role).run
sleep MRSK.config.readiness_delay
execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
rescue SSHKit::Command::Failed => e
if e.message =~ /already in use/
error "Rebooting container with same version #{version} already deployed on #{host} (may cause gap in zero-downtime promise!)"
execute *MRSK.auditor(role: role).record("Rebooted app version #{version}"), verbosity: :debug
execute *MRSK.app(role: role).stop(version: version)
execute *MRSK.app(role: role).remove_container(version: version)
execute *MRSK.app(role: role).run
else
raise
end
end
end
end

View File

@@ -86,6 +86,10 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
docker :ps, "--quiet", *filter_args
end
def container_id_for_version(version)
container_id_for(container_name: container_name(version))
end
def current_running_version
# FIXME: Find more graceful way to extract the version from "app-version" than using sed and tail!
pipe \
@@ -108,6 +112,10 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
xargs(docker(:container, :rm))
end
def rename_container(version:, new_version:)
docker :rename, container_name(version), container_name(new_version)
end
def remove_containers
docker :container, :prune, "--force", *filter_args
end
@@ -126,10 +134,6 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
[ config.service, role, config.destination, version || config.version ].compact.join("-")
end
def container_id_for_version(version)
container_id_for(container_name: container_name(version))
end
def filter_args
argumentize "--filter", filters
end

View File

@@ -16,7 +16,7 @@ module Mrsk::Commands
end
def container_id_for(container_name:)
docker :container, :ls, "--all", "--filter", "name=#{container_name}", "--quiet"
docker :container, :ls, "--all", "--filter", "name=^#{container_name}$", "--quiet"
end
private

View File

@@ -43,7 +43,7 @@ class Mrsk::Commands::Healthcheck < Mrsk::Commands::Base
end
def container_id
container_id_for(container_name: container_name)
container_id_for(container_name: container_name_with_version)
end
def health_url

View File

@@ -26,7 +26,7 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
end
def info
docker :ps, "--filter", "name=traefik"
docker :ps, "--filter", "name=^traefik$"
end
def logs(since: nil, lines: nil, grep: nil)