diff --git a/README.md b/README.md index 1ffd09f1..59bca2dc 100644 --- a/README.md +++ b/README.md @@ -440,11 +440,11 @@ traefik: ### Configure docker options for traefik -We allow users to pass additional docker options to the trafik container like +We allow users to pass additional docker options to the trafik container like ```yaml traefik: - options: + options: publish: - 8080:8080 volumes: @@ -692,6 +692,30 @@ Note that by default old containers are pruned after 3 days when you run `mrsk d If you wish to remove the entire application, including Traefik, containers, images, and registry session, you can run `mrsk remove`. This will leave the servers clean. +## Locking + +Commands that are unsafe to run concurrently will take a deploy lock while they run. The lock is the `mrsk_lock` directory on the primary server. + +You can check the lock status with: + +``` +mrsk lock status + +Locked by: AN Other at 2023-03-24 09:49:03 UTC +Version: 77f45c0686811c68989d6576748475a60bf53fc2 +Message: Automatic deploy lock +``` + +You can also manually acquire and release the lock + +``` +mrsk lock acquire -m "Doing maintanence" +``` + +``` +mrsk lock release +``` + ## Stage of development This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com). diff --git a/lib/mrsk/cli/accessory.rb b/lib/mrsk/cli/accessory.rb index 36f5cd3c..258b4c8d 100644 --- a/lib/mrsk/cli/accessory.rb +++ b/lib/mrsk/cli/accessory.rb @@ -1,34 +1,38 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base desc "boot [NAME]", "Boot new accessory service on host (use NAME=all to boot all accessories)" def boot(name) - if name == "all" - MRSK.accessory_names.each { |accessory_name| boot(accessory_name) } - else - with_accessory(name) do |accessory| - directories(name) - upload(name) + with_lock do + if name == "all" + MRSK.accessory_names.each { |accessory_name| boot(accessory_name) } + else + with_accessory(name) do |accessory| + directories(name) + upload(name) - on(accessory.host) do - execute *MRSK.registry.login - execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug - execute *accessory.run + on(accessory.host) do + execute *MRSK.registry.login + execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug + execute *accessory.run + end + + audit_broadcast "Booted accessory #{name}" unless options[:skip_broadcast] end - - audit_broadcast "Booted accessory #{name}" unless options[:skip_broadcast] end end end desc "upload [NAME]", "Upload accessory files to host", hide: true def upload(name) - with_accessory(name) do |accessory| - on(accessory.host) do - accessory.files.each do |(local, remote)| - accessory.ensure_local_file_present(local) + with_lock do + with_accessory(name) do |accessory| + on(accessory.host) do + accessory.files.each do |(local, remote)| + accessory.ensure_local_file_present(local) - execute *accessory.make_directory_for(remote) - upload! local, remote - execute :chmod, "755", remote + execute *accessory.make_directory_for(remote) + upload! local, remote + execute :chmod, "755", remote + end end end end @@ -36,10 +40,12 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base desc "directories [NAME]", "Create accessory directories on host", hide: true def directories(name) - with_accessory(name) do |accessory| - on(accessory.host) do - accessory.directories.keys.each do |host_path| - execute *accessory.make_directory(host_path) + with_lock do + with_accessory(name) do |accessory| + on(accessory.host) do + accessory.directories.keys.each do |host_path| + execute *accessory.make_directory(host_path) + end end end end @@ -47,38 +53,46 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container)" def reboot(name) - with_accessory(name) do |accessory| - stop(name) - remove_container(name) - boot(name) + with_lock do + with_accessory(name) do |accessory| + stop(name) + remove_container(name) + boot(name) + end end end desc "start [NAME]", "Start existing accessory container on host" def start(name) - with_accessory(name) do |accessory| - on(accessory.host) do - execute *MRSK.auditor.record("Started #{name} accessory"), verbosity: :debug - execute *accessory.start + with_lock do + with_accessory(name) do |accessory| + on(accessory.host) do + execute *MRSK.auditor.record("Started #{name} accessory"), verbosity: :debug + execute *accessory.start + end end end end desc "stop [NAME]", "Stop existing accessory container on host" def stop(name) - with_accessory(name) do |accessory| - on(accessory.host) do - execute *MRSK.auditor.record("Stopped #{name} accessory"), verbosity: :debug - execute *accessory.stop, raise_on_non_zero_exit: false + with_lock do + with_accessory(name) do |accessory| + on(accessory.host) do + execute *MRSK.auditor.record("Stopped #{name} accessory"), verbosity: :debug + execute *accessory.stop, raise_on_non_zero_exit: false + end end end end desc "restart [NAME]", "Restart existing accessory container on host" def restart(name) - with_accessory(name) do - stop(name) - start(name) + with_lock do + with_accessory(name) do + stop(name) + start(name) + end end end diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb index f009d5f7..26c33bbc 100644 --- a/lib/mrsk/cli/app.rb +++ b/lib/mrsk/cli/app.rb @@ -1,32 +1,34 @@ class Mrsk::Cli::App < Mrsk::Cli::Base desc "boot", "Boot app on servers (or reboot app if already running)" def boot - say "Get most recent version available as an image...", :magenta unless options[:version] - using_version(version_or_latest) do |version| - say "Start container with version #{version} using a #{MRSK.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta + with_lock do + say "Get most recent version available as an image...", :magenta unless options[:version] + using_version(version_or_latest) do |version| + say "Start container with version #{version} using a #{MRSK.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta - cli = self + cli = self - MRSK.config.roles.each do |role| - on(role.hosts) do |host| - execute *MRSK.auditor.record("Booted app version #{version}"), verbosity: :debug + MRSK.config.roles.each do |role| + on(role.hosts) do |host| + execute *MRSK.auditor.record("Booted app version #{version}"), verbosity: :debug - begin - old_version = capture_with_info(*MRSK.app.current_running_version).strip - execute *MRSK.app.run(role: role.name) - sleep MRSK.config.readiness_delay - execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present? - - rescue SSHKit::Command::Failed => e - if e.message =~ /already in use/ - error "Rebooting container with same version #{version} already deployed on #{host} (may cause gap in zero-downtime promise!)" - execute *MRSK.auditor.record("Rebooted app version #{version}"), verbosity: :debug - - execute *MRSK.app.stop(version: version) - execute *MRSK.app.remove_container(version: version) + begin + old_version = capture_with_info(*MRSK.app.current_running_version).strip execute *MRSK.app.run(role: role.name) - else - raise + sleep MRSK.config.readiness_delay + execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present? + + rescue SSHKit::Command::Failed => e + if e.message =~ /already in use/ + error "Rebooting container with same version #{version} already deployed on #{host} (may cause gap in zero-downtime promise!)" + execute *MRSK.auditor.record("Rebooted app version #{version}"), verbosity: :debug + + execute *MRSK.app.stop(version: version) + execute *MRSK.app.remove_container(version: version) + execute *MRSK.app.run(role: role.name) + else + raise + end end end end @@ -36,17 +38,21 @@ class Mrsk::Cli::App < Mrsk::Cli::Base desc "start", "Start existing app container on servers" def start - on(MRSK.hosts) do - execute *MRSK.auditor.record("Started app version #{MRSK.config.version}"), verbosity: :debug - execute *MRSK.app.start, raise_on_non_zero_exit: false + with_lock do + on(MRSK.hosts) do + execute *MRSK.auditor.record("Started app version #{MRSK.config.version}"), verbosity: :debug + execute *MRSK.app.start, raise_on_non_zero_exit: false + end end end desc "stop", "Stop app container on servers" def stop - on(MRSK.hosts) do - execute *MRSK.auditor.record("Stopped app"), verbosity: :debug - execute *MRSK.app.stop, raise_on_non_zero_exit: false + with_lock do + on(MRSK.hosts) do + execute *MRSK.auditor.record("Stopped app"), verbosity: :debug + execute *MRSK.app.stop, raise_on_non_zero_exit: false + end end end @@ -140,32 +146,40 @@ class Mrsk::Cli::App < Mrsk::Cli::Base desc "remove", "Remove app containers and images from servers" def remove - stop - remove_containers - remove_images + with_lock do + stop + remove_containers + remove_images + end end desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true def remove_container(version) - on(MRSK.hosts) do - execute *MRSK.auditor.record("Removed app container with version #{version}"), verbosity: :debug - execute *MRSK.app.remove_container(version: version) + with_lock do + on(MRSK.hosts) do + execute *MRSK.auditor.record("Removed app container with version #{version}"), verbosity: :debug + execute *MRSK.app.remove_container(version: version) + end end end desc "remove_containers", "Remove all app containers from servers", hide: true def remove_containers - on(MRSK.hosts) do - execute *MRSK.auditor.record("Removed all app containers"), verbosity: :debug - execute *MRSK.app.remove_containers + with_lock do + on(MRSK.hosts) do + execute *MRSK.auditor.record("Removed all app containers"), verbosity: :debug + execute *MRSK.app.remove_containers + end end end desc "remove_images", "Remove all app images from servers", hide: true def remove_images - on(MRSK.hosts) do - execute *MRSK.auditor.record("Removed all app images"), verbosity: :debug - execute *MRSK.app.remove_images + with_lock do + on(MRSK.hosts) do + execute *MRSK.auditor.record("Removed all app images"), verbosity: :debug + execute *MRSK.app.remove_images + end end end diff --git a/lib/mrsk/cli/base.rb b/lib/mrsk/cli/base.rb index 4e0f2104..ede29a5c 100644 --- a/lib/mrsk/cli/base.rb +++ b/lib/mrsk/cli/base.rb @@ -6,6 +6,8 @@ module Mrsk::Cli class Base < Thor include SSHKit::DSL + class LockError < StandardError; end + def self.exit_on_failure?() true end class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging" @@ -71,5 +73,35 @@ module Mrsk::Cli def audit_broadcast(line) run_locally { execute *MRSK.auditor.broadcast(line), verbosity: :debug } end + + def with_lock + acquire_lock + + yield + ensure + release_lock + end + + def acquire_lock + if MRSK.lock_count == 0 + say "Acquiring the deploy lock" + on(MRSK.primary_host) { execute *MRSK.lock.acquire("Automatic deploy lock", MRSK.config.version) } + end + MRSK.lock_count += 1 + rescue SSHKit::Runner::ExecuteError => e + if e.message =~ /cannot create directory/ + invoke "mrsk:cli:lock:status", [] + end + + raise LockError, "Deploy lock found" + end + + def release_lock + MRSK.lock_count -= 1 + if MRSK.lock_count == 0 + say "Releasing the deploy lock" + on(MRSK.primary_host) { execute *MRSK.lock.release } + end + end end end diff --git a/lib/mrsk/cli/build.rb b/lib/mrsk/cli/build.rb index c9845e4f..289f0bcb 100644 --- a/lib/mrsk/cli/build.rb +++ b/lib/mrsk/cli/build.rb @@ -1,26 +1,30 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base desc "deliver", "Build app and push app image to registry then pull image on servers" def deliver - push - pull + with_lock do + push + pull + end end desc "push", "Build and push app image to registry" def push - cli = self + with_lock do + cli = self - run_locally do - begin - MRSK.with_verbosity(:debug) { execute *MRSK.builder.push } - rescue SSHKit::Command::Failed => e - if e.message =~ /(no builder)|(no such file or directory)/ - error "Missing compatible builder, so creating a new one first" + run_locally do + begin + MRSK.with_verbosity(:debug) { execute *MRSK.builder.push } + rescue SSHKit::Command::Failed => e + if e.message =~ /(no builder)|(no such file or directory)/ + error "Missing compatible builder, so creating a new one first" - if cli.create - MRSK.with_verbosity(:debug) { execute *MRSK.builder.push } + if cli.create + MRSK.with_verbosity(:debug) { execute *MRSK.builder.push } + end + else + raise end - else - raise end end end @@ -28,25 +32,29 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base desc "pull", "Pull app image from registry onto servers" def pull - on(MRSK.hosts) do - execute *MRSK.auditor.record("Pulled image with version #{MRSK.config.version}"), verbosity: :debug - execute *MRSK.builder.clean, raise_on_non_zero_exit: false - execute *MRSK.builder.pull + with_lock do + on(MRSK.hosts) do + execute *MRSK.auditor.record("Pulled image with version #{MRSK.config.version}"), verbosity: :debug + execute *MRSK.builder.clean, raise_on_non_zero_exit: false + execute *MRSK.builder.pull + end end end desc "create", "Create a build setup" def create - run_locally do - begin - debug "Using builder: #{MRSK.builder.name}" - execute *MRSK.builder.create - rescue SSHKit::Command::Failed => e - if e.message =~ /stderr=(.*)/ - error "Couldn't create remote builder: #{$1}" - false - else - raise + with_lock do + run_locally do + begin + debug "Using builder: #{MRSK.builder.name}" + execute *MRSK.builder.create + rescue SSHKit::Command::Failed => e + if e.message =~ /stderr=(.*)/ + error "Couldn't create remote builder: #{$1}" + false + else + raise + end end end end @@ -54,9 +62,11 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base desc "remove", "Remove build setup" def remove - run_locally do - debug "Using builder: #{MRSK.builder.name}" - execute *MRSK.builder.remove + with_lock do + run_locally do + debug "Using builder: #{MRSK.builder.name}" + execute *MRSK.builder.remove + end end end diff --git a/lib/mrsk/cli/lock.rb b/lib/mrsk/cli/lock.rb new file mode 100644 index 00000000..3514e282 --- /dev/null +++ b/lib/mrsk/cli/lock.rb @@ -0,0 +1,37 @@ +class Mrsk::Cli::Lock < Mrsk::Cli::Base + desc "status", "Report lock status" + def status + handle_missing_lock do + on(MRSK.primary_host) { puts capture_with_info(*MRSK.lock.status) } + end + end + + desc "acquire", "Acquire the deploy lock" + option :message, aliases: "-m", type: :string, desc: "A lock mesasge", required: true + def acquire + message = options[:message] + handle_missing_lock do + on(MRSK.primary_host) { execute *MRSK.lock.acquire(message, MRSK.config.version) } + say "Set the deploy lock" + end + end + + desc "release", "Release the deploy lock" + def release + handle_missing_lock do + on(MRSK.primary_host) { execute *MRSK.lock.release } + say "Removed the deploy lock" + end + end + + private + def handle_missing_lock + yield + rescue SSHKit::Runner::ExecuteError => e + if e.message =~ /No such file or directory/ + say "There is no deploy lock" + else + raise + end + end +end diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index 4a3e16bd..a4743220 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -1,96 +1,104 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base desc "setup", "Setup all accessories and deploy app to servers" def setup - print_runtime do - invoke "mrsk:cli:server:bootstrap" - invoke "mrsk:cli:accessory:boot", [ "all" ] - deploy + with_lock do + print_runtime do + invoke "mrsk:cli:server:bootstrap" + invoke "mrsk:cli:accessory:boot", [ "all" ] + deploy + end end end desc "deploy", "Deploy app to servers" option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push" def deploy - invoke_options = deploy_options + with_lock do + invoke_options = deploy_options - runtime = print_runtime do - say "Ensure curl and Docker are installed...", :magenta - invoke "mrsk:cli:server:bootstrap", [], invoke_options + runtime = print_runtime do + say "Ensure curl and Docker are installed...", :magenta + invoke "mrsk:cli:server:bootstrap", [], invoke_options - say "Log into image registry...", :magenta - invoke "mrsk:cli:registry:login", [], invoke_options + say "Log into image registry...", :magenta + invoke "mrsk:cli:registry:login", [], invoke_options - if options[:skip_push] - say "Pull app image...", :magenta - invoke "mrsk:cli:build:pull", [], invoke_options - else - say "Build and push app image...", :magenta - invoke "mrsk:cli:build:deliver", [], invoke_options + if options[:skip_push] + say "Pull app image...", :magenta + invoke "mrsk:cli:build:pull", [], invoke_options + else + say "Build and push app image...", :magenta + invoke "mrsk:cli:build:deliver", [], invoke_options + end + + say "Ensure Traefik is running...", :magenta + invoke "mrsk:cli:traefik:boot", [], invoke_options + + say "Ensure app can pass healthcheck...", :magenta + invoke "mrsk:cli:healthcheck:perform", [], invoke_options + + invoke "mrsk:cli:app:boot", [], invoke_options + + say "Prune old containers and images...", :magenta + invoke "mrsk:cli:prune:all", [], invoke_options end - say "Ensure Traefik is running...", :magenta - invoke "mrsk:cli:traefik:boot", [], invoke_options - - say "Ensure app can pass healthcheck...", :magenta - invoke "mrsk:cli:healthcheck:perform", [], invoke_options - - invoke "mrsk:cli:app:boot", [], invoke_options - - say "Prune old containers and images...", :magenta - invoke "mrsk:cli:prune:all", [], invoke_options + audit_broadcast "Deployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast] end - - audit_broadcast "Deployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast] end desc "redeploy", "Deploy app to servers without bootstrapping servers, starting Traefik, pruning, and registry login" option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push" def redeploy - invoke_options = deploy_options + with_lock do + invoke_options = deploy_options - runtime = print_runtime do - if options[:skip_push] - say "Pull app image...", :magenta - invoke "mrsk:cli:build:pull", [], invoke_options - else - say "Build and push app image...", :magenta - invoke "mrsk:cli:build:deliver", [], invoke_options + runtime = print_runtime do + if options[:skip_push] + say "Pull app image...", :magenta + invoke "mrsk:cli:build:pull", [], invoke_options + else + say "Build and push app image...", :magenta + invoke "mrsk:cli:build:deliver", [], invoke_options + end + + say "Ensure app can pass healthcheck...", :magenta + invoke "mrsk:cli:healthcheck:perform", [], invoke_options + + invoke "mrsk:cli:app:boot", [], invoke_options end - say "Ensure app can pass healthcheck...", :magenta - invoke "mrsk:cli:healthcheck:perform", [], invoke_options - - invoke "mrsk:cli:app:boot", [], invoke_options + audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast] end - - audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast] end desc "rollback [VERSION]", "Rollback app to VERSION" def rollback(version) - MRSK.config.version = version + with_lock do + MRSK.config.version = version - if container_name_available?(MRSK.config.service_with_version) - say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta + if container_name_available?(MRSK.config.service_with_version) + say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta - cli = self - old_version = nil + cli = self + old_version = nil - on(MRSK.hosts) do |host| - old_version = capture_with_info(*MRSK.app.current_running_version).strip.presence + on(MRSK.hosts) do |host| + old_version = capture_with_info(*MRSK.app.current_running_version).strip.presence - execute *MRSK.app.start + execute *MRSK.app.start - if old_version - sleep MRSK.config.readiness_delay + if old_version + sleep MRSK.config.readiness_delay - execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false + execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false + end end - end - audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast] - else - say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red + audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast] + else + say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red + end end end @@ -163,11 +171,13 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base desc "remove", "Remove Traefik, app, accessories, and registry session from servers" option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question" def remove - if options[:confirmed] || ask("This will remove all containers and images. Are you sure?", limited_to: %w( y N ), default: "N") == "y" - invoke "mrsk:cli:traefik:remove", [], options.without(:confirmed) - invoke "mrsk:cli:app:remove", [], options.without(:confirmed) - invoke "mrsk:cli:accessory:remove", [ "all" ], options - invoke "mrsk:cli:registry:logout", [], options.without(:confirmed) + with_lock do + if options[:confirmed] || ask("This will remove all containers and images. Are you sure?", limited_to: %w( y N ), default: "N") == "y" + invoke "mrsk:cli:traefik:remove", [], options.without(:confirmed) + invoke "mrsk:cli:app:remove", [], options.without(:confirmed) + invoke "mrsk:cli:accessory:remove", [ "all" ], options + invoke "mrsk:cli:registry:logout", [], options.without(:confirmed) + end end end @@ -200,6 +210,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base desc "traefik", "Manage Traefik load balancer" subcommand "traefik", Mrsk::Cli::Traefik + desc "lock", "Manage the deploy lock" + subcommand "lock", Mrsk::Cli::Lock + private def container_name_available?(container_name, host: MRSK.primary_host) container_names = nil diff --git a/lib/mrsk/cli/prune.rb b/lib/mrsk/cli/prune.rb index fb23060e..6698ba27 100644 --- a/lib/mrsk/cli/prune.rb +++ b/lib/mrsk/cli/prune.rb @@ -1,23 +1,29 @@ class Mrsk::Cli::Prune < Mrsk::Cli::Base desc "all", "Prune unused images and stopped containers" def all - containers - images + with_lock do + containers + images + end end desc "images", "Prune unused images older than 7 days" def images - on(MRSK.hosts) do - execute *MRSK.auditor.record("Pruned images"), verbosity: :debug - execute *MRSK.prune.images + with_lock do + on(MRSK.hosts) do + execute *MRSK.auditor.record("Pruned images"), verbosity: :debug + execute *MRSK.prune.images + end end end desc "containers", "Prune stopped containers older than 3 days" def containers - on(MRSK.hosts) do - execute *MRSK.auditor.record("Pruned containers"), verbosity: :debug - execute *MRSK.prune.containers + with_lock do + on(MRSK.hosts) do + execute *MRSK.auditor.record("Pruned containers"), verbosity: :debug + execute *MRSK.prune.containers + end end end end diff --git a/lib/mrsk/cli/server.rb b/lib/mrsk/cli/server.rb index ffd9a3cb..0ede5afc 100644 --- a/lib/mrsk/cli/server.rb +++ b/lib/mrsk/cli/server.rb @@ -1,14 +1,16 @@ class Mrsk::Cli::Server < Mrsk::Cli::Base desc "bootstrap", "Ensure curl and Docker are installed on servers" def bootstrap - on(MRSK.hosts + MRSK.accessory_hosts) do - dependencies_to_install = Array.new.tap do |dependencies| - dependencies << "curl" unless execute "which curl", raise_on_non_zero_exit: false - dependencies << "docker.io" unless execute "which docker", raise_on_non_zero_exit: false - end + with_lock do + on(MRSK.hosts + MRSK.accessory_hosts) do + dependencies_to_install = Array.new.tap do |dependencies| + dependencies << "curl" unless execute "which curl", raise_on_non_zero_exit: false + dependencies << "docker.io" unless execute "which docker", raise_on_non_zero_exit: false + end - if dependencies_to_install.any? - execute "apt-get update -y && apt-get install #{dependencies_to_install.join(" ")} -y" + if dependencies_to_install.any? + execute "apt-get update -y && apt-get install #{dependencies_to_install.join(" ")} -y" + end end end end diff --git a/lib/mrsk/cli/traefik.rb b/lib/mrsk/cli/traefik.rb index af9a3cb3..7be64e8e 100644 --- a/lib/mrsk/cli/traefik.rb +++ b/lib/mrsk/cli/traefik.rb @@ -1,36 +1,46 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base desc "boot", "Boot Traefik on servers" def boot - on(MRSK.traefik_hosts) { execute *MRSK.traefik.run, raise_on_non_zero_exit: false } + with_lock do + on(MRSK.traefik_hosts) { execute *MRSK.traefik.run, raise_on_non_zero_exit: false } + end end desc "reboot", "Reboot Traefik on servers (stop container, remove container, start new container)" def reboot - stop - remove_container - boot + with_lock do + stop + remove_container + boot + end end desc "start", "Start existing Traefik container on servers" def start - on(MRSK.traefik_hosts) do - execute *MRSK.auditor.record("Started traefik"), verbosity: :debug - execute *MRSK.traefik.start, raise_on_non_zero_exit: false + with_lock do + on(MRSK.traefik_hosts) do + execute *MRSK.auditor.record("Started traefik"), verbosity: :debug + execute *MRSK.traefik.start, raise_on_non_zero_exit: false + end end end desc "stop", "Stop existing Traefik container on servers" def stop - on(MRSK.traefik_hosts) do - execute *MRSK.auditor.record("Stopped traefik"), verbosity: :debug - execute *MRSK.traefik.stop, raise_on_non_zero_exit: false + with_lock do + on(MRSK.traefik_hosts) do + execute *MRSK.auditor.record("Stopped traefik"), verbosity: :debug + execute *MRSK.traefik.stop, raise_on_non_zero_exit: false + end end end desc "restart", "Restart existing Traefik container on servers" def restart - stop - start + with_lock do + stop + start + end end desc "details", "Show details about Traefik container from servers" @@ -64,24 +74,30 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base desc "remove", "Remove Traefik container and image from servers" def remove - stop - remove_container - remove_image + with_lock do + stop + remove_container + remove_image + end end desc "remove_container", "Remove Traefik container from servers", hide: true def remove_container - on(MRSK.traefik_hosts) do - execute *MRSK.auditor.record("Removed traefik container"), verbosity: :debug - execute *MRSK.traefik.remove_container + with_lock do + on(MRSK.traefik_hosts) do + execute *MRSK.auditor.record("Removed traefik container"), verbosity: :debug + execute *MRSK.traefik.remove_container + end end end desc "remove_container", "Remove Traefik image from servers", hide: true def remove_image - on(MRSK.traefik_hosts) do - execute *MRSK.auditor.record("Removed traefik image"), verbosity: :debug - execute *MRSK.traefik.remove_image + with_lock do + on(MRSK.traefik_hosts) do + execute *MRSK.auditor.record("Removed traefik image"), verbosity: :debug + execute *MRSK.traefik.remove_image + end end end end diff --git a/lib/mrsk/commander.rb b/lib/mrsk/commander.rb index 5aa24fc7..abcc48f5 100644 --- a/lib/mrsk/commander.rb +++ b/lib/mrsk/commander.rb @@ -2,10 +2,11 @@ require "active_support/core_ext/enumerable" require "active_support/core_ext/module/delegation" class Mrsk::Commander - attr_accessor :verbosity + attr_accessor :verbosity, :lock_count def initialize self.verbosity = :info + self.lock_count = 0 end @@ -84,6 +85,9 @@ class Mrsk::Commander @traefik ||= Mrsk::Commands::Traefik.new(config) end + def lock + @lock ||= Mrsk::Commands::Lock.new(config) + end def with_verbosity(level) old_level = self.verbosity @@ -97,14 +101,6 @@ class Mrsk::Commander SSHKit.config.output_verbosity = old_level end - - # Test-induced damage! - def reset - @config = nil - @app = @builder = @traefik = @registry = @prune = @auditor = nil - @verbosity = :info - end - private # Lazy setup of SSHKit def configure_sshkit_with(config) diff --git a/lib/mrsk/commands/base.rb b/lib/mrsk/commands/base.rb index d84f4dc3..df1c4305 100644 --- a/lib/mrsk/commands/base.rb +++ b/lib/mrsk/commands/base.rb @@ -41,6 +41,10 @@ module Mrsk::Commands combine *commands, by: ">>" end + def write(*commands) + combine *commands, by: ">" + end + def xargs(command) [ :xargs, command ].flatten end diff --git a/lib/mrsk/commands/lock.rb b/lib/mrsk/commands/lock.rb new file mode 100644 index 00000000..c8870238 --- /dev/null +++ b/lib/mrsk/commands/lock.rb @@ -0,0 +1,63 @@ +require "active_support/duration" +require "active_support/core_ext/numeric/time" + +class Mrsk::Commands::Lock < Mrsk::Commands::Base + def acquire(message, version) + combine \ + [:mkdir, lock_dir], + write_lock_details(message, version) + end + + def release + combine \ + [:rm, lock_details_file], + [:rm, "-r", lock_dir] + end + + def status + combine \ + stat_lock_dir, + read_lock_details + end + + private + def write_lock_details(message, version) + write \ + [:echo, "\"#{Base64.encode64(lock_details(message, version))}\""], + lock_details_file + end + + def read_lock_details + pipe \ + [:cat, lock_details_file], + [:base64, "-d"] + end + + def stat_lock_dir + write \ + [:stat, lock_dir], + "/dev/null" + end + + def lock_dir + :mrsk_lock + end + + def lock_details_file + [lock_dir, :details].join("/") + end + + def lock_details(message, version) + <<~DETAILS.strip + Locked by: #{locked_by} at #{Time.now.gmtime} + Version: #{version} + Message: #{message} + DETAILS + end + + def locked_by + `git config user.name`.strip + rescue Errno::ENOENT + "Unknown" + end +end diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb index d721373b..1c06034b 100644 --- a/test/cli/build_test.rb +++ b/test/cli/build_test.rb @@ -15,8 +15,9 @@ class CliBuildTest < CliTestCase end test "push without builder" do - Mrsk::Cli::Build.any_instance.stubs(:create).returns(true) + stub_locking SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with { |arg| arg == :docker } .raises(SSHKit::Command::Failed.new("no builder")) .then .returns(true) @@ -40,7 +41,9 @@ class CliBuildTest < CliTestCase end test "create with error" do + stub_locking SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with { |arg| arg == :docker } .raises(SSHKit::Command::Failed.new("stderr=error")) run_command("create").tap do |output| @@ -69,4 +72,11 @@ class CliBuildTest < CliTestCase def run_command(*command) stdouted { Mrsk::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) } end + + def stub_locking + SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock } + SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" } + end end diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb index f7a0ad07..496d6a27 100644 --- a/test/cli/cli_test_case.rb +++ b/test/cli/cli_test_case.rb @@ -8,13 +8,14 @@ class CliTestCase < ActiveSupport::TestCase ENV["VERSION"] = "999" ENV["RAILS_MASTER_KEY"] = "123" ENV["MYSQL_ROOT_PASSWORD"] = "secret123" + Object.send(:remove_const, :MRSK) + Object.const_set(:MRSK, Mrsk::Commander.new) end teardown do ENV.delete("RAILS_MASTER_KEY") ENV.delete("MYSQL_ROOT_PASSWORD") ENV.delete("VERSION") - MRSK.reset end private diff --git a/test/cli/lock_test.rb b/test/cli/lock_test.rb new file mode 100644 index 00000000..a5c45c2c --- /dev/null +++ b/test/cli/lock_test.rb @@ -0,0 +1,20 @@ +require_relative "cli_test_case" + +class CliLockTest < CliTestCase + test "status" do + run_command("status") do |output| + assert_match "stat lock", output + end + end + + test "release" do + run_command("release") do |output| + assert_match "rm -rf lock", output + end + end + + private + def run_command(*command) + stdouted { Mrsk::Cli::Lock.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) } + end +end diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index d4aa3d82..3a506378 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -42,12 +42,14 @@ class CliMainTest < CliTestCase Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options) run_command("deploy", "--skip_push").tap do |output| + assert_match /Acquiring the deploy lock/, output assert_match /Ensure curl and Docker are installed/, output assert_match /Log into image registry/, output assert_match /Pull app image/, output assert_match /Ensure Traefik is running/, output assert_match /Ensure app can pass healthcheck/, output assert_match /Prune old containers and images/, output + assert_match /Releasing the deploy lock/, output end end diff --git a/test/commands/lock_test.rb b/test/commands/lock_test.rb new file mode 100644 index 00000000..bfb76597 --- /dev/null +++ b/test/commands/lock_test.rb @@ -0,0 +1,33 @@ +require "test_helper" + +class CommandsLockTest < ActiveSupport::TestCase + setup do + @config = { + service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], + traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } } + } + end + + test "status" do + assert_equal \ + "stat mrsk_lock > /dev/null && cat mrsk_lock/details | base64 -d", + new_command.status.join(" ") + end + + test "acquire" do + assert_match \ + /mkdir mrsk_lock && echo ".*" > mrsk_lock\/details/m, + new_command.acquire("Hello", "123").join(" ") + end + + test "release" do + assert_match \ + "rm mrsk_lock/details && rm -r mrsk_lock", + new_command.release.join(" ") + end + + private + def new_command + Mrsk::Commands::Lock.new(Mrsk::Configuration.new(@config, version: "123")) + end +end