spacebar/kamal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Deploy locks

Browse Source 
Add a deploy lock for commands that are unsafe to run concurrently.

The lock is taken by creating a `mrsk_lock` directory on the primary
host. Details of who took the lock are added to a details file in that
directory.

Additional CLI commands have been added to manual release and acquire
the lock and to check its status.

```
Commands:
  mrsk lock acquire -m, --message=MESSAGE  # Acquire the deploy lock
  mrsk lock help [COMMAND]                 # Describe subcommands or one specific subcommand
  mrsk lock release                        # Release the deploy lock
  mrsk lock status                         # Report lock status

Options:
  -v, [--verbose], [--no-verbose]                # Detailed logging
  -q, [--quiet], [--no-quiet]                    # Minimal logging
      [--version=VERSION]                        # Run commands against a specific app version
  -p, [--primary], [--no-primary]                # Run commands only on primary host instead of all
  -h, [--hosts=HOSTS]                            # Run commands on these hosts instead of all (separate by comma)
  -r, [--roles=ROLES]                            # Run commands on these roles instead of all (separate by comma)
  -c, [--config-file=CONFIG_FILE]                # Path to config file
                                                 # Default: config/deploy.yml
  -d, [--destination=DESTINATION]                # Specify destination to be used for config file (staging -> deploy.staging.yml)
  -B, [--skip-broadcast], [--no-skip-broadcast]  # Skip audit broadcasts
```

If we add support for running multiple deployments on a single server
we'll need to extend the locking to lock per deployment.
This commit is contained in:
Donal McBreen
2023-03-23 16:58:49 +00:00
parent 17e74910e4
commit 8d8f9f6ada
18 changed files with 516 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
README.md
View File

@@ -440,11 +440,11 @@ traefik:
### Configure docker options for traefik
We allow users to pass additional docker options to the trafik container like
We allow users to pass additional docker options to the trafik container like
```yaml
traefik:
options:
options:
publish:
- 8080:8080
volumes:
@@ -692,6 +692,30 @@ Note that by default old containers are pruned after 3 days when you run `mrsk d
If you wish to remove the entire application, including Traefik, containers, images, and registry session, you can run `mrsk remove`. This will leave the servers clean.
## Locking
Commands that are unsafe to run concurrently will take a deploy lock while they run. The lock is the `mrsk_lock` directory on the primary server.
You can check the lock status with:
```
mrsk lock status
Locked by: AN Other at 2023-03-24 09:49:03 UTC
Version: 77f45c0686811c68989d6576748475a60bf53fc2
Message: Automatic deploy lock
```
You can also manually acquire and release the lock
```
mrsk lock acquire -m "Doing maintanence"
```
```
mrsk lock release
```
## Stage of development
This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com).

90
lib/mrsk/cli/accessory.rb
View File

@@ -1,34 +1,38 @@
class Mrsk::Cli::Accessory < Mrsk::Cli::Base
desc "boot [NAME]", "Boot new accessory service on host (use NAME=all to boot all accessories)"
def boot(name)
if name == "all"
MRSK.accessory_names.each { |accessory_name| boot(accessory_name) }
else
with_accessory(name) do |accessory|
directories(name)
upload(name)
with_lock do
if name == "all"
MRSK.accessory_names.each { |accessory_name| boot(accessory_name) }
else
with_accessory(name) do |accessory|
directories(name)
upload(name)
on(accessory.host) do
execute *MRSK.registry.login
execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug
execute *accessory.run
on(accessory.host) do
execute *MRSK.registry.login
execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug
execute *accessory.run
end
audit_broadcast "Booted accessory #{name}" unless options[:skip_broadcast]
end
audit_broadcast "Booted accessory #{name}" unless options[:skip_broadcast]
end
end
end
desc "upload [NAME]", "Upload accessory files to host", hide: true
def upload(name)
with_accessory(name) do |accessory|
on(accessory.host) do
accessory.files.each do |(local, remote)|
accessory.ensure_local_file_present(local)
with_lock do
with_accessory(name) do |accessory|
on(accessory.host) do
accessory.files.each do |(local, remote)|
accessory.ensure_local_file_present(local)
execute *accessory.make_directory_for(remote)
upload! local, remote
execute :chmod, "755", remote
execute *accessory.make_directory_for(remote)
upload! local, remote
execute :chmod, "755", remote
end
end
end
end
@@ -36,10 +40,12 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
desc "directories [NAME]", "Create accessory directories on host", hide: true
def directories(name)
with_accessory(name) do |accessory|
on(accessory.host) do
accessory.directories.keys.each do |host_path|
execute *accessory.make_directory(host_path)
with_lock do
with_accessory(name) do |accessory|
on(accessory.host) do
accessory.directories.keys.each do |host_path|
execute *accessory.make_directory(host_path)
end
end
end
end
@@ -47,38 +53,46 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container)"
def reboot(name)
with_accessory(name) do |accessory|
stop(name)
remove_container(name)
boot(name)
with_lock do
with_accessory(name) do |accessory|
stop(name)
remove_container(name)
boot(name)
end
end
end
desc "start [NAME]", "Start existing accessory container on host"
def start(name)
with_accessory(name) do |accessory|
on(accessory.host) do
execute *MRSK.auditor.record("Started #{name} accessory"), verbosity: :debug
execute *accessory.start
with_lock do
with_accessory(name) do |accessory|
on(accessory.host) do
execute *MRSK.auditor.record("Started #{name} accessory"), verbosity: :debug
execute *accessory.start
end
end
end
end
desc "stop [NAME]", "Stop existing accessory container on host"
def stop(name)
with_accessory(name) do |accessory|
on(accessory.host) do
execute *MRSK.auditor.record("Stopped #{name} accessory"), verbosity: :debug
execute *accessory.stop, raise_on_non_zero_exit: false
with_lock do
with_accessory(name) do |accessory|
on(accessory.host) do
execute *MRSK.auditor.record("Stopped #{name} accessory"), verbosity: :debug
execute *accessory.stop, raise_on_non_zero_exit: false
end
end
end
end
desc "restart [NAME]", "Restart existing accessory container on host"
def restart(name)
with_accessory(name) do
stop(name)
start(name)
with_lock do
with_accessory(name) do
stop(name)
start(name)
end
end
end

94
lib/mrsk/cli/app.rb
View File

@@ -1,32 +1,34 @@
class Mrsk::Cli::App < Mrsk::Cli::Base
desc "boot", "Boot app on servers (or reboot app if already running)"
def boot
say "Get most recent version available as an image...", :magenta unless options[:version]
using_version(version_or_latest) do |version|
say "Start container with version #{version} using a #{MRSK.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
with_lock do
say "Get most recent version available as an image...", :magenta unless options[:version]
using_version(version_or_latest) do |version|
say "Start container with version #{version} using a #{MRSK.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
cli = self
cli = self
MRSK.config.roles.each do |role|
on(role.hosts) do |host|
execute *MRSK.auditor.record("Booted app version #{version}"), verbosity: :debug
MRSK.config.roles.each do |role|
on(role.hosts) do |host|
execute *MRSK.auditor.record("Booted app version #{version}"), verbosity: :debug
begin
old_version = capture_with_info(*MRSK.app.current_running_version).strip
execute *MRSK.app.run(role: role.name)
sleep MRSK.config.readiness_delay
execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
rescue SSHKit::Command::Failed => e
if e.message =~ /already in use/
error "Rebooting container with same version #{version} already deployed on #{host} (may cause gap in zero-downtime promise!)"
execute *MRSK.auditor.record("Rebooted app version #{version}"), verbosity: :debug
execute *MRSK.app.stop(version: version)
execute *MRSK.app.remove_container(version: version)
begin
old_version = capture_with_info(*MRSK.app.current_running_version).strip
execute *MRSK.app.run(role: role.name)
else
raise
sleep MRSK.config.readiness_delay
execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
rescue SSHKit::Command::Failed => e
if e.message =~ /already in use/
error "Rebooting container with same version #{version} already deployed on #{host} (may cause gap in zero-downtime promise!)"
execute *MRSK.auditor.record("Rebooted app version #{version}"), verbosity: :debug
execute *MRSK.app.stop(version: version)
execute *MRSK.app.remove_container(version: version)
execute *MRSK.app.run(role: role.name)
else
raise
end
end
end
end
@@ -36,17 +38,21 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
desc "start", "Start existing app container on servers"
def start
on(MRSK.hosts) do
execute *MRSK.auditor.record("Started app version #{MRSK.config.version}"), verbosity: :debug
execute *MRSK.app.start, raise_on_non_zero_exit: false
with_lock do
on(MRSK.hosts) do
execute *MRSK.auditor.record("Started app version #{MRSK.config.version}"), verbosity: :debug
execute *MRSK.app.start, raise_on_non_zero_exit: false
end
end
end
desc "stop", "Stop app container on servers"
def stop
on(MRSK.hosts) do
execute *MRSK.auditor.record("Stopped app"), verbosity: :debug
execute *MRSK.app.stop, raise_on_non_zero_exit: false
with_lock do
on(MRSK.hosts) do
execute *MRSK.auditor.record("Stopped app"), verbosity: :debug
execute *MRSK.app.stop, raise_on_non_zero_exit: false
end
end
end
@@ -140,32 +146,40 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
desc "remove", "Remove app containers and images from servers"
def remove
stop
remove_containers
remove_images
with_lock do
stop
remove_containers
remove_images
end
end
desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true
def remove_container(version)
on(MRSK.hosts) do
execute *MRSK.auditor.record("Removed app container with version #{version}"), verbosity: :debug
execute *MRSK.app.remove_container(version: version)
with_lock do
on(MRSK.hosts) do
execute *MRSK.auditor.record("Removed app container with version #{version}"), verbosity: :debug
execute *MRSK.app.remove_container(version: version)
end
end
end
desc "remove_containers", "Remove all app containers from servers", hide: true
def remove_containers
on(MRSK.hosts) do
execute *MRSK.auditor.record("Removed all app containers"), verbosity: :debug
execute *MRSK.app.remove_containers
with_lock do
on(MRSK.hosts) do
execute *MRSK.auditor.record("Removed all app containers"), verbosity: :debug
execute *MRSK.app.remove_containers
end
end
end
desc "remove_images", "Remove all app images from servers", hide: true
def remove_images
on(MRSK.hosts) do
execute *MRSK.auditor.record("Removed all app images"), verbosity: :debug
execute *MRSK.app.remove_images
with_lock do
on(MRSK.hosts) do
execute *MRSK.auditor.record("Removed all app images"), verbosity: :debug
execute *MRSK.app.remove_images
end
end
end

32
lib/mrsk/cli/base.rb
View File

@@ -6,6 +6,8 @@ module Mrsk::Cli
class Base < Thor
include SSHKit::DSL
class LockError < StandardError; end
def self.exit_on_failure?() true end
class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging"
@@ -71,5 +73,35 @@ module Mrsk::Cli
def audit_broadcast(line)
run_locally { execute *MRSK.auditor.broadcast(line), verbosity: :debug }
end
def with_lock
acquire_lock
yield
ensure
release_lock
end
def acquire_lock
if MRSK.lock_count == 0
say "Acquiring the deploy lock"
on(MRSK.primary_host) { execute *MRSK.lock.acquire("Automatic deploy lock", MRSK.config.version) }
end
MRSK.lock_count += 1
rescue SSHKit::Runner::ExecuteError => e
if e.message =~ /cannot create directory/
invoke "mrsk:cli:lock:status", []
end
raise LockError, "Deploy lock found"
end
def release_lock
MRSK.lock_count -= 1
if MRSK.lock_count == 0
say "Releasing the deploy lock"
on(MRSK.primary_host) { execute *MRSK.lock.release }
end
end
end
end

70
lib/mrsk/cli/build.rb
View File

@@ -1,26 +1,30 @@
class Mrsk::Cli::Build < Mrsk::Cli::Base
desc "deliver", "Build app and push app image to registry then pull image on servers"
def deliver
push
pull
with_lock do
push
pull
end
end
desc "push", "Build and push app image to registry"
def push
cli = self
with_lock do
cli = self
run_locally do
begin
MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
rescue SSHKit::Command::Failed => e
if e.message =~ /(no builder)|(no such file or directory)/
error "Missing compatible builder, so creating a new one first"
run_locally do
begin
MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
rescue SSHKit::Command::Failed => e
if e.message =~ /(no builder)|(no such file or directory)/
error "Missing compatible builder, so creating a new one first"
if cli.create
MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
if cli.create
MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
end
else
raise
end
else
raise
end
end
end
@@ -28,25 +32,29 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
desc "pull", "Pull app image from registry onto servers"
def pull
on(MRSK.hosts) do
execute *MRSK.auditor.record("Pulled image with version #{MRSK.config.version}"), verbosity: :debug
execute *MRSK.builder.clean, raise_on_non_zero_exit: false
execute *MRSK.builder.pull
with_lock do
on(MRSK.hosts) do
execute *MRSK.auditor.record("Pulled image with version #{MRSK.config.version}"), verbosity: :debug
execute *MRSK.builder.clean, raise_on_non_zero_exit: false
execute *MRSK.builder.pull
end
end
end
desc "create", "Create a build setup"
def create
run_locally do
begin
debug "Using builder: #{MRSK.builder.name}"
execute *MRSK.builder.create
rescue SSHKit::Command::Failed => e
if e.message =~ /stderr=(.*)/
error "Couldn't create remote builder: #{$1}"
false
else
raise
with_lock do
run_locally do
begin
debug "Using builder: #{MRSK.builder.name}"
execute *MRSK.builder.create
rescue SSHKit::Command::Failed => e
if e.message =~ /stderr=(.*)/
error "Couldn't create remote builder: #{$1}"
false
else
raise
end
end
end
end
@@ -54,9 +62,11 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
desc "remove", "Remove build setup"
def remove
run_locally do
debug "Using builder: #{MRSK.builder.name}"
execute *MRSK.builder.remove
with_lock do
run_locally do
debug "Using builder: #{MRSK.builder.name}"
execute *MRSK.builder.remove
end
end
end

37
lib/mrsk/cli/lock.rb Normal file
View File

@@ -0,0 +1,37 @@
class Mrsk::Cli::Lock < Mrsk::Cli::Base
desc "status", "Report lock status"
def status
handle_missing_lock do
on(MRSK.primary_host) { puts capture_with_info(*MRSK.lock.status) }
end
end
desc "acquire", "Acquire the deploy lock"
option :message, aliases: "-m", type: :string, desc: "A lock mesasge", required: true
def acquire
message = options[:message]
handle_missing_lock do
on(MRSK.primary_host) { execute *MRSK.lock.acquire(message, MRSK.config.version) }
say "Set the deploy lock"
end
end
desc "release", "Release the deploy lock"
def release
handle_missing_lock do
on(MRSK.primary_host) { execute *MRSK.lock.release }
say "Removed the deploy lock"
end
end
private
def handle_missing_lock
yield
rescue SSHKit::Runner::ExecuteError => e
if e.message =~ /No such file or directory/
say "There is no deploy lock"
else
raise
end
end
end

137
lib/mrsk/cli/main.rb
View File

@@ -1,96 +1,104 @@
class Mrsk::Cli::Main < Mrsk::Cli::Base
desc "setup", "Setup all accessories and deploy app to servers"
def setup
print_runtime do
invoke "mrsk:cli:server:bootstrap"
invoke "mrsk:cli:accessory:boot", [ "all" ]
deploy
with_lock do
print_runtime do
invoke "mrsk:cli:server:bootstrap"
invoke "mrsk:cli:accessory:boot", [ "all" ]
deploy
end
end
end
desc "deploy", "Deploy app to servers"
option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
def deploy
invoke_options = deploy_options
with_lock do
invoke_options = deploy_options
runtime = print_runtime do
say "Ensure curl and Docker are installed...", :magenta
invoke "mrsk:cli:server:bootstrap", [], invoke_options
runtime = print_runtime do
say "Ensure curl and Docker are installed...", :magenta
invoke "mrsk:cli:server:bootstrap", [], invoke_options
say "Log into image registry...", :magenta
invoke "mrsk:cli:registry:login", [], invoke_options
say "Log into image registry...", :magenta
invoke "mrsk:cli:registry:login", [], invoke_options
if options[:skip_push]
say "Pull app image...", :magenta
invoke "mrsk:cli:build:pull", [], invoke_options
else
say "Build and push app image...", :magenta
invoke "mrsk:cli:build:deliver", [], invoke_options
if options[:skip_push]
say "Pull app image...", :magenta
invoke "mrsk:cli:build:pull", [], invoke_options
else
say "Build and push app image...", :magenta
invoke "mrsk:cli:build:deliver", [], invoke_options
end
say "Ensure Traefik is running...", :magenta
invoke "mrsk:cli:traefik:boot", [], invoke_options
say "Ensure app can pass healthcheck...", :magenta
invoke "mrsk:cli:healthcheck:perform", [], invoke_options
invoke "mrsk:cli:app:boot", [], invoke_options
say "Prune old containers and images...", :magenta
invoke "mrsk:cli:prune:all", [], invoke_options
end
say "Ensure Traefik is running...", :magenta
invoke "mrsk:cli:traefik:boot", [], invoke_options
say "Ensure app can pass healthcheck...", :magenta
invoke "mrsk:cli:healthcheck:perform", [], invoke_options
invoke "mrsk:cli:app:boot", [], invoke_options
say "Prune old containers and images...", :magenta
invoke "mrsk:cli:prune:all", [], invoke_options
audit_broadcast "Deployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
end
audit_broadcast "Deployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
end
desc "redeploy", "Deploy app to servers without bootstrapping servers, starting Traefik, pruning, and registry login"
option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
def redeploy
invoke_options = deploy_options
with_lock do
invoke_options = deploy_options
runtime = print_runtime do
if options[:skip_push]
say "Pull app image...", :magenta
invoke "mrsk:cli:build:pull", [], invoke_options
else
say "Build and push app image...", :magenta
invoke "mrsk:cli:build:deliver", [], invoke_options
runtime = print_runtime do
if options[:skip_push]
say "Pull app image...", :magenta
invoke "mrsk:cli:build:pull", [], invoke_options
else
say "Build and push app image...", :magenta
invoke "mrsk:cli:build:deliver", [], invoke_options
end
say "Ensure app can pass healthcheck...", :magenta
invoke "mrsk:cli:healthcheck:perform", [], invoke_options
invoke "mrsk:cli:app:boot", [], invoke_options
end
say "Ensure app can pass healthcheck...", :magenta
invoke "mrsk:cli:healthcheck:perform", [], invoke_options
invoke "mrsk:cli:app:boot", [], invoke_options
audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
end
audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
end
desc "rollback [VERSION]", "Rollback app to VERSION"
def rollback(version)
MRSK.config.version = version
with_lock do
MRSK.config.version = version
if container_name_available?(MRSK.config.service_with_version)
say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta
if container_name_available?(MRSK.config.service_with_version)
say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta
cli = self
old_version = nil
cli = self
old_version = nil
on(MRSK.hosts) do |host|
old_version = capture_with_info(*MRSK.app.current_running_version).strip.presence
on(MRSK.hosts) do |host|
old_version = capture_with_info(*MRSK.app.current_running_version).strip.presence
execute *MRSK.app.start
execute *MRSK.app.start
if old_version
sleep MRSK.config.readiness_delay
if old_version
sleep MRSK.config.readiness_delay
execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false
execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false
end
end
end
audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast]
else
say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast]
else
say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
end
end
end
@@ -163,11 +171,13 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
desc "remove", "Remove Traefik, app, accessories, and registry session from servers"
option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
def remove
if options[:confirmed] || ask("This will remove all containers and images. Are you sure?", limited_to: %w( y N ), default: "N") == "y"
invoke "mrsk:cli:traefik:remove", [], options.without(:confirmed)
invoke "mrsk:cli:app:remove", [], options.without(:confirmed)
invoke "mrsk:cli:accessory:remove", [ "all" ], options
invoke "mrsk:cli:registry:logout", [], options.without(:confirmed)
with_lock do
if options[:confirmed] || ask("This will remove all containers and images. Are you sure?", limited_to: %w( y N ), default: "N") == "y"
invoke "mrsk:cli:traefik:remove", [], options.without(:confirmed)
invoke "mrsk:cli:app:remove", [], options.without(:confirmed)
invoke "mrsk:cli:accessory:remove", [ "all" ], options
invoke "mrsk:cli:registry:logout", [], options.without(:confirmed)
end
end
end
@@ -200,6 +210,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
desc "traefik", "Manage Traefik load balancer"
subcommand "traefik", Mrsk::Cli::Traefik
desc "lock", "Manage the deploy lock"
subcommand "lock", Mrsk::Cli::Lock
private
def container_name_available?(container_name, host: MRSK.primary_host)
container_names = nil

22
lib/mrsk/cli/prune.rb
View File

@@ -1,23 +1,29 @@
class Mrsk::Cli::Prune < Mrsk::Cli::Base
desc "all", "Prune unused images and stopped containers"
def all
containers
images
with_lock do
containers
images
end
end
desc "images", "Prune unused images older than 7 days"
def images
on(MRSK.hosts) do
execute *MRSK.auditor.record("Pruned images"), verbosity: :debug
execute *MRSK.prune.images
with_lock do
on(MRSK.hosts) do
execute *MRSK.auditor.record("Pruned images"), verbosity: :debug
execute *MRSK.prune.images
end
end
end
desc "containers", "Prune stopped containers older than 3 days"
def containers
on(MRSK.hosts) do
execute *MRSK.auditor.record("Pruned containers"), verbosity: :debug
execute *MRSK.prune.containers
with_lock do
on(MRSK.hosts) do
execute *MRSK.auditor.record("Pruned containers"), verbosity: :debug
execute *MRSK.prune.containers
end
end
end
end

16
lib/mrsk/cli/server.rb
View File

@@ -1,14 +1,16 @@
class Mrsk::Cli::Server < Mrsk::Cli::Base
desc "bootstrap", "Ensure curl and Docker are installed on servers"
def bootstrap
on(MRSK.hosts + MRSK.accessory_hosts) do
dependencies_to_install = Array.new.tap do |dependencies|
dependencies << "curl" unless execute "which curl", raise_on_non_zero_exit: false
dependencies << "docker.io" unless execute "which docker", raise_on_non_zero_exit: false
end
with_lock do
on(MRSK.hosts + MRSK.accessory_hosts) do
dependencies_to_install = Array.new.tap do |dependencies|
dependencies << "curl" unless execute "which curl", raise_on_non_zero_exit: false
dependencies << "docker.io" unless execute "which docker", raise_on_non_zero_exit: false
end
if dependencies_to_install.any?
execute "apt-get update -y && apt-get install #{dependencies_to_install.join(" ")} -y"
if dependencies_to_install.any?
execute "apt-get update -y && apt-get install #{dependencies_to_install.join(" ")} -y"
end
end
end
end

58
lib/mrsk/cli/traefik.rb
View File

@@ -1,36 +1,46 @@
class Mrsk::Cli::Traefik < Mrsk::Cli::Base
desc "boot", "Boot Traefik on servers"
def boot
on(MRSK.traefik_hosts) { execute *MRSK.traefik.run, raise_on_non_zero_exit: false }
with_lock do
on(MRSK.traefik_hosts) { execute *MRSK.traefik.run, raise_on_non_zero_exit: false }
end
end
desc "reboot", "Reboot Traefik on servers (stop container, remove container, start new container)"
def reboot
stop
remove_container
boot
with_lock do
stop
remove_container
boot
end
end
desc "start", "Start existing Traefik container on servers"
def start
on(MRSK.traefik_hosts) do
execute *MRSK.auditor.record("Started traefik"), verbosity: :debug
execute *MRSK.traefik.start, raise_on_non_zero_exit: false
with_lock do
on(MRSK.traefik_hosts) do
execute *MRSK.auditor.record("Started traefik"), verbosity: :debug
execute *MRSK.traefik.start, raise_on_non_zero_exit: false
end
end
end
desc "stop", "Stop existing Traefik container on servers"
def stop
on(MRSK.traefik_hosts) do
execute *MRSK.auditor.record("Stopped traefik"), verbosity: :debug
execute *MRSK.traefik.stop, raise_on_non_zero_exit: false
with_lock do
on(MRSK.traefik_hosts) do
execute *MRSK.auditor.record("Stopped traefik"), verbosity: :debug
execute *MRSK.traefik.stop, raise_on_non_zero_exit: false
end
end
end
desc "restart", "Restart existing Traefik container on servers"
def restart
stop
start
with_lock do
stop
start
end
end
desc "details", "Show details about Traefik container from servers"
@@ -64,24 +74,30 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
desc "remove", "Remove Traefik container and image from servers"
def remove
stop
remove_container
remove_image
with_lock do
stop
remove_container
remove_image
end
end
desc "remove_container", "Remove Traefik container from servers", hide: true
def remove_container
on(MRSK.traefik_hosts) do
execute *MRSK.auditor.record("Removed traefik container"), verbosity: :debug
execute *MRSK.traefik.remove_container
with_lock do
on(MRSK.traefik_hosts) do
execute *MRSK.auditor.record("Removed traefik container"), verbosity: :debug
execute *MRSK.traefik.remove_container
end
end
end
desc "remove_container", "Remove Traefik image from servers", hide: true
def remove_image
on(MRSK.traefik_hosts) do
execute *MRSK.auditor.record("Removed traefik image"), verbosity: :debug
execute *MRSK.traefik.remove_image
with_lock do
on(MRSK.traefik_hosts) do
execute *MRSK.auditor.record("Removed traefik image"), verbosity: :debug
execute *MRSK.traefik.remove_image
end
end
end
end

14
lib/mrsk/commander.rb
View File

@@ -2,10 +2,11 @@ require "active_support/core_ext/enumerable"
require "active_support/core_ext/module/delegation"
class Mrsk::Commander
attr_accessor :verbosity
attr_accessor :verbosity, :lock_count
def initialize
self.verbosity = :info
self.lock_count = 0
end
@@ -84,6 +85,9 @@ class Mrsk::Commander
@traefik ||= Mrsk::Commands::Traefik.new(config)
end
def lock
@lock ||= Mrsk::Commands::Lock.new(config)
end
def with_verbosity(level)
old_level = self.verbosity
@@ -97,14 +101,6 @@ class Mrsk::Commander
SSHKit.config.output_verbosity = old_level
end
# Test-induced damage!
def reset
@config = nil
@app = @builder = @traefik = @registry = @prune = @auditor = nil
@verbosity = :info
end
private
# Lazy setup of SSHKit
def configure_sshkit_with(config)

4
lib/mrsk/commands/base.rb
View File

@@ -41,6 +41,10 @@ module Mrsk::Commands
combine *commands, by: ">>"
end
def write(*commands)
combine *commands, by: ">"
end
def xargs(command)
[ :xargs, command ].flatten
end

63
lib/mrsk/commands/lock.rb Normal file
View File

@@ -0,0 +1,63 @@
require "active_support/duration"
require "active_support/core_ext/numeric/time"
class Mrsk::Commands::Lock < Mrsk::Commands::Base
def acquire(message, version)
combine \
[:mkdir, lock_dir],
write_lock_details(message, version)
end
def release
combine \
[:rm, lock_details_file],
[:rm, "-r", lock_dir]
end
def status
combine \
stat_lock_dir,
read_lock_details
end
private
def write_lock_details(message, version)
write \
[:echo, "\"#{Base64.encode64(lock_details(message, version))}\""],
lock_details_file
end
def read_lock_details
pipe \
[:cat, lock_details_file],
[:base64, "-d"]
end
def stat_lock_dir
write \
[:stat, lock_dir],
"/dev/null"
end
def lock_dir
:mrsk_lock
end
def lock_details_file
[lock_dir, :details].join("/")
end
def lock_details(message, version)
<<~DETAILS.strip
Locked by: #{locked_by} at #{Time.now.gmtime}
Version: #{version}
Message: #{message}
DETAILS
end
def locked_by
`git config user.name`.strip
rescue Errno::ENOENT
"Unknown"
end
end

12
test/cli/build_test.rb
View File

@@ -15,8 +15,9 @@ class CliBuildTest < CliTestCase
end
test "push without builder" do
Mrsk::Cli::Build.any_instance.stubs(:create).returns(true)
stub_locking
SSHKit::Backend::Abstract.any_instance.stubs(:execute)
.with { |arg| arg == :docker }
.raises(SSHKit::Command::Failed.new("no builder"))
.then
.returns(true)
@@ -40,7 +41,9 @@ class CliBuildTest < CliTestCase
end
test "create with error" do
stub_locking
SSHKit::Backend::Abstract.any_instance.stubs(:execute)
.with { |arg| arg == :docker }
.raises(SSHKit::Command::Failed.new("stderr=error"))
run_command("create").tap do |output|
@@ -69,4 +72,11 @@ class CliBuildTest < CliTestCase
def run_command(*command)
stdouted { Mrsk::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
end
def stub_locking
SSHKit::Backend::Abstract.any_instance.stubs(:execute)
.with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock }
SSHKit::Backend::Abstract.any_instance.stubs(:execute)
.with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" }
end
end

3
test/cli/cli_test_case.rb
View File

@@ -8,13 +8,14 @@ class CliTestCase < ActiveSupport::TestCase
ENV["VERSION"] = "999"
ENV["RAILS_MASTER_KEY"] = "123"
ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
Object.send(:remove_const, :MRSK)
Object.const_set(:MRSK, Mrsk::Commander.new)
end
teardown do
ENV.delete("RAILS_MASTER_KEY")
ENV.delete("MYSQL_ROOT_PASSWORD")
ENV.delete("VERSION")
MRSK.reset
end
private

20
test/cli/lock_test.rb Normal file
View File

@@ -0,0 +1,20 @@
require_relative "cli_test_case"
class CliLockTest < CliTestCase
test "status" do
run_command("status") do |output|
assert_match "stat lock", output
end
end
test "release" do
run_command("release") do |output|
assert_match "rm -rf lock", output
end
end
private
def run_command(*command)
stdouted { Mrsk::Cli::Lock.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
end
end

2
test/cli/main_test.rb
View File

@@ -42,12 +42,14 @@ class CliMainTest < CliTestCase
Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
run_command("deploy", "--skip_push").tap do |output|
assert_match /Acquiring the deploy lock/, output
assert_match /Ensure curl and Docker are installed/, output
assert_match /Log into image registry/, output
assert_match /Pull app image/, output
assert_match /Ensure Traefik is running/, output
assert_match /Ensure app can pass healthcheck/, output
assert_match /Prune old containers and images/, output
assert_match /Releasing the deploy lock/, output
end
end

33
test/commands/lock_test.rb Normal file
View File

@@ -0,0 +1,33 @@
require "test_helper"
class CommandsLockTest < ActiveSupport::TestCase
setup do
@config = {
service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
}
end
test "status" do
assert_equal \
"stat mrsk_lock > /dev/null && cat mrsk_lock/details | base64 -d",
new_command.status.join(" ")
end
test "acquire" do
assert_match \
/mkdir mrsk_lock && echo ".*" > mrsk_lock\/details/m,
new_command.acquire("Hello", "123").join(" ")
end
test "release" do
assert_match \
"rm mrsk_lock/details && rm -r mrsk_lock",
new_command.release.join(" ")
end
private
def new_command
Mrsk::Commands::Lock.new(Mrsk::Configuration.new(@config, version: "123"))
end
end