Deploy locks

Add a deploy lock for commands that are unsafe to run concurrently.

The lock is taken by creating a `mrsk_lock` directory on the primary
host. Details of who took the lock are added to a details file in that
directory.

Additional CLI commands have been added to manual release and acquire
the lock and to check its status.

```
Commands:
  mrsk lock acquire -m, --message=MESSAGE  # Acquire the deploy lock
  mrsk lock help [COMMAND]                 # Describe subcommands or one specific subcommand
  mrsk lock release                        # Release the deploy lock
  mrsk lock status                         # Report lock status

Options:
  -v, [--verbose], [--no-verbose]                # Detailed logging
  -q, [--quiet], [--no-quiet]                    # Minimal logging
      [--version=VERSION]                        # Run commands against a specific app version
  -p, [--primary], [--no-primary]                # Run commands only on primary host instead of all
  -h, [--hosts=HOSTS]                            # Run commands on these hosts instead of all (separate by comma)
  -r, [--roles=ROLES]                            # Run commands on these roles instead of all (separate by comma)
  -c, [--config-file=CONFIG_FILE]                # Path to config file
                                                 # Default: config/deploy.yml
  -d, [--destination=DESTINATION]                # Specify destination to be used for config file (staging -> deploy.staging.yml)
  -B, [--skip-broadcast], [--no-skip-broadcast]  # Skip audit broadcasts
```

If we add support for running multiple deployments on a single server
we'll need to extend the locking to lock per deployment.

README.md

@@ -440,11 +440,11 @@ traefik:
 
 ### Configure docker options for traefik
 
-We allow users to pass additional docker options to the trafik container like 
+We allow users to pass additional docker options to the trafik container like
 
 ```yaml
 traefik:
-  options: 
+  options:
     publish:
     - 8080:8080
     volumes:
@@ -692,6 +692,30 @@ Note that by default old containers are pruned after 3 days when you run `mrsk d
 
 If you wish to remove the entire application, including Traefik, containers, images, and registry session, you can run `mrsk remove`. This will leave the servers clean.
 
+## Locking
+
+Commands that are unsafe to run concurrently will take a deploy lock while they run. The lock is the `mrsk_lock` directory on the primary server.
+
+You can check the lock status with:
+
+```
+mrsk lock status
+
+Locked by: AN Other at 2023-03-24 09:49:03 UTC
+Version: 77f45c0686811c68989d6576748475a60bf53fc2
+Message: Automatic deploy lock
+```
+
+You can also manually acquire and release the lock
+
+```
+mrsk lock acquire -m "Doing maintanence"
+```
+
+```
+mrsk lock release
+```
+
 ## Stage of development
 
 This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com).

lib/mrsk/cli/accessory.rb

@@ -1,34 +1,38 @@
 class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   desc "boot [NAME]", "Boot new accessory service on host (use NAME=all to boot all accessories)"
   def boot(name)
-    if name == "all"
+    with_lock do
-      MRSK.accessory_names.each { |accessory_name| boot(accessory_name) }
+      if name == "all"
-    else
+        MRSK.accessory_names.each { |accessory_name| boot(accessory_name) }
-      with_accessory(name) do |accessory|
+      else
-        directories(name)
+        with_accessory(name) do |accessory|
-        upload(name)
+          directories(name)
+          upload(name)
 
-        on(accessory.host) do
+          on(accessory.host) do
-          execute *MRSK.registry.login
+            execute *MRSK.registry.login
-          execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug
+            execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug
-          execute *accessory.run
+            execute *accessory.run
+          end
+
+          audit_broadcast "Booted accessory #{name}" unless options[:skip_broadcast]
         end
-
-        audit_broadcast "Booted accessory #{name}" unless options[:skip_broadcast]
       end
     end
   end
 
   desc "upload [NAME]", "Upload accessory files to host", hide: true
   def upload(name)
-    with_accessory(name) do |accessory|
+    with_lock do
-      on(accessory.host) do
+      with_accessory(name) do |accessory|
-        accessory.files.each do |(local, remote)|
+        on(accessory.host) do
-          accessory.ensure_local_file_present(local)
+          accessory.files.each do |(local, remote)|
+            accessory.ensure_local_file_present(local)
 
-          execute *accessory.make_directory_for(remote)
+            execute *accessory.make_directory_for(remote)
-          upload! local, remote
+            upload! local, remote
-          execute :chmod, "755", remote
+            execute :chmod, "755", remote
+          end
         end
       end
     end
@@ -36,10 +40,12 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
 
   desc "directories [NAME]", "Create accessory directories on host", hide: true
   def directories(name)
-    with_accessory(name) do |accessory|
+    with_lock do
-      on(accessory.host) do
+      with_accessory(name) do |accessory|
-        accessory.directories.keys.each do |host_path|
+        on(accessory.host) do
-          execute *accessory.make_directory(host_path)
+          accessory.directories.keys.each do |host_path|
+            execute *accessory.make_directory(host_path)
+          end
         end
       end
     end
@@ -47,38 +53,46 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
 
   desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container)"
   def reboot(name)
-    with_accessory(name) do |accessory|
+    with_lock do
-      stop(name)
+      with_accessory(name) do |accessory|
-      remove_container(name)
+        stop(name)
-      boot(name)
+        remove_container(name)
+        boot(name)
+      end
     end
   end
 
   desc "start [NAME]", "Start existing accessory container on host"
   def start(name)
-    with_accessory(name) do |accessory|
+    with_lock do
-      on(accessory.host) do
+      with_accessory(name) do |accessory|
-        execute *MRSK.auditor.record("Started #{name} accessory"), verbosity: :debug
+        on(accessory.host) do
-        execute *accessory.start
+          execute *MRSK.auditor.record("Started #{name} accessory"), verbosity: :debug
+          execute *accessory.start
+        end
       end
     end
   end
 
   desc "stop [NAME]", "Stop existing accessory container on host"
   def stop(name)
-    with_accessory(name) do |accessory|
+    with_lock do
-      on(accessory.host) do
+      with_accessory(name) do |accessory|
-        execute *MRSK.auditor.record("Stopped #{name} accessory"), verbosity: :debug
+        on(accessory.host) do
-        execute *accessory.stop, raise_on_non_zero_exit: false
+          execute *MRSK.auditor.record("Stopped #{name} accessory"), verbosity: :debug
+          execute *accessory.stop, raise_on_non_zero_exit: false
+        end
       end
     end
   end
 
   desc "restart [NAME]", "Restart existing accessory container on host"
   def restart(name)
-    with_accessory(name) do
+    with_lock do
-      stop(name)
+      with_accessory(name) do
-      start(name)
+        stop(name)
+        start(name)
+      end
     end
   end
 

lib/mrsk/cli/app.rb

@@ -1,32 +1,34 @@
 class Mrsk::Cli::App < Mrsk::Cli::Base
   desc "boot", "Boot app on servers (or reboot app if already running)"
   def boot
-    say "Get most recent version available as an image...", :magenta unless options[:version]
+    with_lock do
-    using_version(version_or_latest) do |version|
+      say "Get most recent version available as an image...", :magenta unless options[:version]
-      say "Start container with version #{version} using a #{MRSK.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
+      using_version(version_or_latest) do |version|
+        say "Start container with version #{version} using a #{MRSK.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
 
-      cli = self
+        cli = self
 
-      MRSK.config.roles.each do |role|
+        MRSK.config.roles.each do |role|
-        on(role.hosts) do |host|
+          on(role.hosts) do |host|
-          execute *MRSK.auditor.record("Booted app version #{version}"), verbosity: :debug
+            execute *MRSK.auditor.record("Booted app version #{version}"), verbosity: :debug
 
-          begin
+            begin
-            old_version = capture_with_info(*MRSK.app.current_running_version).strip
+              old_version = capture_with_info(*MRSK.app.current_running_version).strip
-            execute *MRSK.app.run(role: role.name)
-            sleep MRSK.config.readiness_delay
-            execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
-
-          rescue SSHKit::Command::Failed => e
-            if e.message =~ /already in use/
-              error "Rebooting container with same version #{version} already deployed on #{host} (may cause gap in zero-downtime promise!)"
-              execute *MRSK.auditor.record("Rebooted app version #{version}"), verbosity: :debug
-
-              execute *MRSK.app.stop(version: version)
-              execute *MRSK.app.remove_container(version: version)
               execute *MRSK.app.run(role: role.name)
-            else
+              sleep MRSK.config.readiness_delay
-              raise
+              execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
+
+            rescue SSHKit::Command::Failed => e
+              if e.message =~ /already in use/
+                error "Rebooting container with same version #{version} already deployed on #{host} (may cause gap in zero-downtime promise!)"
+                execute *MRSK.auditor.record("Rebooted app version #{version}"), verbosity: :debug
+
+                execute *MRSK.app.stop(version: version)
+                execute *MRSK.app.remove_container(version: version)
+                execute *MRSK.app.run(role: role.name)
+              else
+                raise
+              end
             end
           end
         end
@@ -36,17 +38,21 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
 
   desc "start", "Start existing app container on servers"
   def start
-    on(MRSK.hosts) do
+    with_lock do
-      execute *MRSK.auditor.record("Started app version #{MRSK.config.version}"), verbosity: :debug
+      on(MRSK.hosts) do
-      execute *MRSK.app.start, raise_on_non_zero_exit: false
+        execute *MRSK.auditor.record("Started app version #{MRSK.config.version}"), verbosity: :debug
+        execute *MRSK.app.start, raise_on_non_zero_exit: false
+      end
     end
   end
 
   desc "stop", "Stop app container on servers"
   def stop
-    on(MRSK.hosts) do
+    with_lock do
-      execute *MRSK.auditor.record("Stopped app"), verbosity: :debug
+      on(MRSK.hosts) do
-      execute *MRSK.app.stop, raise_on_non_zero_exit: false
+        execute *MRSK.auditor.record("Stopped app"), verbosity: :debug
+        execute *MRSK.app.stop, raise_on_non_zero_exit: false
+      end
     end
   end
 
@@ -140,32 +146,40 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
 
   desc "remove", "Remove app containers and images from servers"
   def remove
-    stop
+    with_lock do
-    remove_containers
+      stop
-    remove_images
+      remove_containers
+      remove_images
+    end
   end
 
   desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true
   def remove_container(version)
-    on(MRSK.hosts) do
+    with_lock do
-      execute *MRSK.auditor.record("Removed app container with version #{version}"), verbosity: :debug
+      on(MRSK.hosts) do
-      execute *MRSK.app.remove_container(version: version)
+        execute *MRSK.auditor.record("Removed app container with version #{version}"), verbosity: :debug
+        execute *MRSK.app.remove_container(version: version)
+      end
     end
   end
 
   desc "remove_containers", "Remove all app containers from servers", hide: true
   def remove_containers
-    on(MRSK.hosts) do
+    with_lock do
-      execute *MRSK.auditor.record("Removed all app containers"), verbosity: :debug
+      on(MRSK.hosts) do
-      execute *MRSK.app.remove_containers
+        execute *MRSK.auditor.record("Removed all app containers"), verbosity: :debug
+        execute *MRSK.app.remove_containers
+      end
     end
   end
 
   desc "remove_images", "Remove all app images from servers", hide: true
   def remove_images
-    on(MRSK.hosts) do
+    with_lock do
-      execute *MRSK.auditor.record("Removed all app images"), verbosity: :debug
+      on(MRSK.hosts) do
-      execute *MRSK.app.remove_images
+        execute *MRSK.auditor.record("Removed all app images"), verbosity: :debug
+        execute *MRSK.app.remove_images
+      end
     end
   end
 

lib/mrsk/cli/base.rb

@@ -6,6 +6,8 @@ module Mrsk::Cli
   class Base < Thor
     include SSHKit::DSL
 
+    class LockError < StandardError; end
+
     def self.exit_on_failure?() true end
 
     class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging"
@@ -71,5 +73,35 @@ module Mrsk::Cli
       def audit_broadcast(line)
         run_locally { execute *MRSK.auditor.broadcast(line), verbosity: :debug }
       end
+
+      def with_lock
+        acquire_lock
+
+        yield
+      ensure
+        release_lock
+      end
+
+      def acquire_lock
+        if MRSK.lock_count == 0
+          say "Acquiring the deploy lock"
+          on(MRSK.primary_host) { execute *MRSK.lock.acquire("Automatic deploy lock", MRSK.config.version) }
+        end
+        MRSK.lock_count += 1
+      rescue SSHKit::Runner::ExecuteError => e
+        if e.message =~ /cannot create directory/
+          invoke "mrsk:cli:lock:status", []
+        end
+
+        raise LockError, "Deploy lock found"
+      end
+
+      def release_lock
+        MRSK.lock_count -= 1
+        if MRSK.lock_count == 0
+          say "Releasing the deploy lock"
+          on(MRSK.primary_host) { execute *MRSK.lock.release }
+        end
+      end
   end
 end

lib/mrsk/cli/build.rb

@@ -1,26 +1,30 @@
 class Mrsk::Cli::Build < Mrsk::Cli::Base
   desc "deliver", "Build app and push app image to registry then pull image on servers"
   def deliver
-    push
+    with_lock do
-    pull
+      push
+      pull
+    end
   end
 
   desc "push", "Build and push app image to registry"
   def push
-    cli = self
+    with_lock do
+      cli = self
 
-    run_locally do
+      run_locally do
-      begin
+        begin
-        MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
+          MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
-      rescue SSHKit::Command::Failed => e
+        rescue SSHKit::Command::Failed => e
-        if e.message =~ /(no builder)|(no such file or directory)/
+          if e.message =~ /(no builder)|(no such file or directory)/
-          error "Missing compatible builder, so creating a new one first"
+            error "Missing compatible builder, so creating a new one first"
 
-          if cli.create
+            if cli.create
-            MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
+              MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
+            end
+          else
+            raise
           end
-        else
-          raise
         end
       end
     end
@@ -28,25 +32,29 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
 
   desc "pull", "Pull app image from registry onto servers"
   def pull
-    on(MRSK.hosts) do
+    with_lock do
-      execute *MRSK.auditor.record("Pulled image with version #{MRSK.config.version}"), verbosity: :debug
+      on(MRSK.hosts) do
-      execute *MRSK.builder.clean, raise_on_non_zero_exit: false
+        execute *MRSK.auditor.record("Pulled image with version #{MRSK.config.version}"), verbosity: :debug
-      execute *MRSK.builder.pull
+        execute *MRSK.builder.clean, raise_on_non_zero_exit: false
+        execute *MRSK.builder.pull
+      end
     end
   end
 
   desc "create", "Create a build setup"
   def create
-    run_locally do
+    with_lock do
-      begin
+      run_locally do
-        debug "Using builder: #{MRSK.builder.name}"
+        begin
-        execute *MRSK.builder.create
+          debug "Using builder: #{MRSK.builder.name}"
-      rescue SSHKit::Command::Failed => e
+          execute *MRSK.builder.create
-        if e.message =~ /stderr=(.*)/
+        rescue SSHKit::Command::Failed => e
-          error "Couldn't create remote builder: #{$1}"
+          if e.message =~ /stderr=(.*)/
-          false
+            error "Couldn't create remote builder: #{$1}"
-        else
+            false
-          raise
+          else
+            raise
+          end
         end
       end
     end
@@ -54,9 +62,11 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
 
   desc "remove", "Remove build setup"
   def remove
-    run_locally do
+    with_lock do
-      debug "Using builder: #{MRSK.builder.name}"
+      run_locally do
-      execute *MRSK.builder.remove
+        debug "Using builder: #{MRSK.builder.name}"
+        execute *MRSK.builder.remove
+      end
     end
   end
 

lib/mrsk/cli/lock.rb

@@ -0,0 +1,37 @@
+class Mrsk::Cli::Lock < Mrsk::Cli::Base
+  desc "status", "Report lock status"
+  def status
+    handle_missing_lock do
+      on(MRSK.primary_host) { puts capture_with_info(*MRSK.lock.status) }
+    end
+  end
+
+  desc "acquire", "Acquire the deploy lock"
+  option :message, aliases: "-m", type: :string, desc: "A lock mesasge", required: true
+  def acquire
+    message = options[:message]
+    handle_missing_lock do
+      on(MRSK.primary_host) { execute *MRSK.lock.acquire(message, MRSK.config.version) }
+      say "Set the deploy lock"
+    end
+  end
+
+  desc "release", "Release the deploy lock"
+  def release
+    handle_missing_lock do
+      on(MRSK.primary_host) { execute *MRSK.lock.release }
+      say "Removed the deploy lock"
+    end
+  end
+
+  private
+    def handle_missing_lock
+      yield
+    rescue SSHKit::Runner::ExecuteError => e
+      if e.message =~ /No such file or directory/
+        say "There is no deploy lock"
+      else
+        raise
+      end
+    end
+end

lib/mrsk/cli/main.rb

@@ -1,96 +1,104 @@
 class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "setup", "Setup all accessories and deploy app to servers"
   def setup
-    print_runtime do
+    with_lock do
-      invoke "mrsk:cli:server:bootstrap"
+      print_runtime do
-      invoke "mrsk:cli:accessory:boot", [ "all" ]
+        invoke "mrsk:cli:server:bootstrap"
-      deploy
+        invoke "mrsk:cli:accessory:boot", [ "all" ]
+        deploy
+      end
     end
   end
 
   desc "deploy", "Deploy app to servers"
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def deploy
-    invoke_options = deploy_options
+    with_lock do
+      invoke_options = deploy_options
 
-    runtime = print_runtime do
+      runtime = print_runtime do
-      say "Ensure curl and Docker are installed...", :magenta
+        say "Ensure curl and Docker are installed...", :magenta
-      invoke "mrsk:cli:server:bootstrap", [], invoke_options
+        invoke "mrsk:cli:server:bootstrap", [], invoke_options
 
-      say "Log into image registry...", :magenta
+        say "Log into image registry...", :magenta
-      invoke "mrsk:cli:registry:login", [], invoke_options
+        invoke "mrsk:cli:registry:login", [], invoke_options
 
-      if options[:skip_push]
+        if options[:skip_push]
-        say "Pull app image...", :magenta
+          say "Pull app image...", :magenta
-        invoke "mrsk:cli:build:pull", [], invoke_options
+          invoke "mrsk:cli:build:pull", [], invoke_options
-      else
+        else
-        say "Build and push app image...", :magenta
+          say "Build and push app image...", :magenta
-        invoke "mrsk:cli:build:deliver", [], invoke_options
+          invoke "mrsk:cli:build:deliver", [], invoke_options
+        end
+
+        say "Ensure Traefik is running...", :magenta
+        invoke "mrsk:cli:traefik:boot", [], invoke_options
+
+        say "Ensure app can pass healthcheck...", :magenta
+        invoke "mrsk:cli:healthcheck:perform", [], invoke_options
+
+        invoke "mrsk:cli:app:boot", [], invoke_options
+
+        say "Prune old containers and images...", :magenta
+        invoke "mrsk:cli:prune:all", [], invoke_options
       end
 
-      say "Ensure Traefik is running...", :magenta
+      audit_broadcast "Deployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
-      invoke "mrsk:cli:traefik:boot", [], invoke_options
-
-      say "Ensure app can pass healthcheck...", :magenta
-      invoke "mrsk:cli:healthcheck:perform", [], invoke_options
-
-      invoke "mrsk:cli:app:boot", [], invoke_options
-
-      say "Prune old containers and images...", :magenta
-      invoke "mrsk:cli:prune:all", [], invoke_options
     end
-
-    audit_broadcast "Deployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
   end
 
   desc "redeploy", "Deploy app to servers without bootstrapping servers, starting Traefik, pruning, and registry login"
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def redeploy
-    invoke_options = deploy_options
+    with_lock do
+      invoke_options = deploy_options
 
-    runtime = print_runtime do
+      runtime = print_runtime do
-      if options[:skip_push]
+        if options[:skip_push]
-        say "Pull app image...", :magenta
+          say "Pull app image...", :magenta
-        invoke "mrsk:cli:build:pull", [], invoke_options
+          invoke "mrsk:cli:build:pull", [], invoke_options
-      else
+        else
-        say "Build and push app image...", :magenta
+          say "Build and push app image...", :magenta
-        invoke "mrsk:cli:build:deliver", [], invoke_options
+          invoke "mrsk:cli:build:deliver", [], invoke_options
+        end
+
+        say "Ensure app can pass healthcheck...", :magenta
+        invoke "mrsk:cli:healthcheck:perform", [], invoke_options
+
+        invoke "mrsk:cli:app:boot", [], invoke_options
       end
 
-      say "Ensure app can pass healthcheck...", :magenta
+      audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
-      invoke "mrsk:cli:healthcheck:perform", [], invoke_options
-
-      invoke "mrsk:cli:app:boot", [], invoke_options
     end
-
-    audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
   end
 
   desc "rollback [VERSION]", "Rollback app to VERSION"
   def rollback(version)
-    MRSK.config.version = version
+    with_lock do
+      MRSK.config.version = version
 
-    if container_name_available?(MRSK.config.service_with_version)
+      if container_name_available?(MRSK.config.service_with_version)
-      say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta
+        say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta
 
-      cli = self
+        cli = self
-      old_version = nil
+        old_version = nil
 
-      on(MRSK.hosts) do |host|
+        on(MRSK.hosts) do |host|
-        old_version = capture_with_info(*MRSK.app.current_running_version).strip.presence
+          old_version = capture_with_info(*MRSK.app.current_running_version).strip.presence
 
-        execute *MRSK.app.start
+          execute *MRSK.app.start
 
-        if old_version
+          if old_version
-          sleep MRSK.config.readiness_delay
+            sleep MRSK.config.readiness_delay
 
-          execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false
+            execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false
+          end
         end
-      end
 
-      audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast]
+        audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast]
-    else
+      else
-      say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
+        say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
+      end
     end
   end
 
@@ -163,11 +171,13 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "remove", "Remove Traefik, app, accessories, and registry session from servers"
   option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def remove
-    if options[:confirmed] || ask("This will remove all containers and images. Are you sure?", limited_to: %w( y N ), default: "N") == "y"
+    with_lock do
-      invoke "mrsk:cli:traefik:remove", [], options.without(:confirmed)
+      if options[:confirmed] || ask("This will remove all containers and images. Are you sure?", limited_to: %w( y N ), default: "N") == "y"
-      invoke "mrsk:cli:app:remove", [], options.without(:confirmed)
+        invoke "mrsk:cli:traefik:remove", [], options.without(:confirmed)
-      invoke "mrsk:cli:accessory:remove", [ "all" ], options
+        invoke "mrsk:cli:app:remove", [], options.without(:confirmed)
-      invoke "mrsk:cli:registry:logout", [], options.without(:confirmed)
+        invoke "mrsk:cli:accessory:remove", [ "all" ], options
+        invoke "mrsk:cli:registry:logout", [], options.without(:confirmed)
+      end
     end
   end
 
@@ -200,6 +210,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "traefik", "Manage Traefik load balancer"
   subcommand "traefik", Mrsk::Cli::Traefik
 
+  desc "lock", "Manage the deploy lock"
+  subcommand "lock", Mrsk::Cli::Lock
+
   private
     def container_name_available?(container_name, host: MRSK.primary_host)
       container_names = nil

lib/mrsk/cli/prune.rb

@@ -1,23 +1,29 @@
 class Mrsk::Cli::Prune < Mrsk::Cli::Base
   desc "all", "Prune unused images and stopped containers"
   def all
-    containers
+    with_lock do
-    images
+      containers
+      images
+    end
   end
 
   desc "images", "Prune unused images older than 7 days"
   def images
-    on(MRSK.hosts) do
+    with_lock do
-      execute *MRSK.auditor.record("Pruned images"), verbosity: :debug
+      on(MRSK.hosts) do
-      execute *MRSK.prune.images
+        execute *MRSK.auditor.record("Pruned images"), verbosity: :debug
+        execute *MRSK.prune.images
+      end
     end
   end
 
   desc "containers", "Prune stopped containers older than 3 days"
   def containers
-    on(MRSK.hosts) do
+    with_lock do
-      execute *MRSK.auditor.record("Pruned containers"), verbosity: :debug
+      on(MRSK.hosts) do
-      execute *MRSK.prune.containers
+        execute *MRSK.auditor.record("Pruned containers"), verbosity: :debug
+        execute *MRSK.prune.containers
+      end
     end
   end
 end

lib/mrsk/cli/server.rb

@@ -1,14 +1,16 @@
 class Mrsk::Cli::Server < Mrsk::Cli::Base
   desc "bootstrap", "Ensure curl and Docker are installed on servers"
   def bootstrap
-    on(MRSK.hosts + MRSK.accessory_hosts) do
+    with_lock do
-      dependencies_to_install = Array.new.tap do |dependencies|
+      on(MRSK.hosts + MRSK.accessory_hosts) do
-        dependencies << "curl" unless execute "which curl", raise_on_non_zero_exit: false
+        dependencies_to_install = Array.new.tap do |dependencies|
-        dependencies << "docker.io" unless execute "which docker", raise_on_non_zero_exit: false
+          dependencies << "curl" unless execute "which curl", raise_on_non_zero_exit: false
-      end
+          dependencies << "docker.io" unless execute "which docker", raise_on_non_zero_exit: false
+        end
 
-      if dependencies_to_install.any?
+        if dependencies_to_install.any?
-        execute "apt-get update -y && apt-get install #{dependencies_to_install.join(" ")} -y"
+          execute "apt-get update -y && apt-get install #{dependencies_to_install.join(" ")} -y"
+        end
       end
     end
   end

lib/mrsk/cli/traefik.rb

@@ -1,36 +1,46 @@
 class Mrsk::Cli::Traefik < Mrsk::Cli::Base
   desc "boot", "Boot Traefik on servers"
   def boot
-    on(MRSK.traefik_hosts) { execute *MRSK.traefik.run, raise_on_non_zero_exit: false }
+    with_lock do
+      on(MRSK.traefik_hosts) { execute *MRSK.traefik.run, raise_on_non_zero_exit: false }
+    end
   end
 
   desc "reboot", "Reboot Traefik on servers (stop container, remove container, start new container)"
   def reboot
-    stop
+    with_lock do
-    remove_container
+      stop
-    boot
+      remove_container
+      boot
+    end
   end
 
   desc "start", "Start existing Traefik container on servers"
   def start
-    on(MRSK.traefik_hosts) do
+    with_lock do
-      execute *MRSK.auditor.record("Started traefik"), verbosity: :debug
+      on(MRSK.traefik_hosts) do
-      execute *MRSK.traefik.start, raise_on_non_zero_exit: false
+        execute *MRSK.auditor.record("Started traefik"), verbosity: :debug
+        execute *MRSK.traefik.start, raise_on_non_zero_exit: false
+      end
     end
   end
 
   desc "stop", "Stop existing Traefik container on servers"
   def stop
-    on(MRSK.traefik_hosts) do
+    with_lock do
-      execute *MRSK.auditor.record("Stopped traefik"), verbosity: :debug
+      on(MRSK.traefik_hosts) do
-      execute *MRSK.traefik.stop, raise_on_non_zero_exit: false
+        execute *MRSK.auditor.record("Stopped traefik"), verbosity: :debug
+        execute *MRSK.traefik.stop, raise_on_non_zero_exit: false
+      end
     end
   end
 
   desc "restart", "Restart existing Traefik container on servers"
   def restart
-    stop
+    with_lock do
-    start
+      stop
+      start
+    end
   end
 
   desc "details", "Show details about Traefik container from servers"
@@ -64,24 +74,30 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
 
   desc "remove", "Remove Traefik container and image from servers"
   def remove
-    stop
+    with_lock do
-    remove_container
+      stop
-    remove_image
+      remove_container
+      remove_image
+    end
   end
 
   desc "remove_container", "Remove Traefik container from servers", hide: true
   def remove_container
-    on(MRSK.traefik_hosts) do
+    with_lock do
-      execute *MRSK.auditor.record("Removed traefik container"), verbosity: :debug
+      on(MRSK.traefik_hosts) do
-      execute *MRSK.traefik.remove_container
+        execute *MRSK.auditor.record("Removed traefik container"), verbosity: :debug
+        execute *MRSK.traefik.remove_container
+      end
     end
   end
 
   desc "remove_container", "Remove Traefik image from servers", hide: true
   def remove_image
-    on(MRSK.traefik_hosts) do
+    with_lock do
-      execute *MRSK.auditor.record("Removed traefik image"), verbosity: :debug
+      on(MRSK.traefik_hosts) do
-      execute *MRSK.traefik.remove_image
+        execute *MRSK.auditor.record("Removed traefik image"), verbosity: :debug
+        execute *MRSK.traefik.remove_image
+      end
     end
   end
 end

lib/mrsk/commander.rb

@@ -2,10 +2,11 @@ require "active_support/core_ext/enumerable"
 require "active_support/core_ext/module/delegation"
 
 class Mrsk::Commander
-  attr_accessor :verbosity
+  attr_accessor :verbosity, :lock_count
 
   def initialize
     self.verbosity = :info
+    self.lock_count = 0
   end
 
 
@@ -84,6 +85,9 @@ class Mrsk::Commander
     @traefik ||= Mrsk::Commands::Traefik.new(config)
   end
 
+  def lock
+    @lock ||= Mrsk::Commands::Lock.new(config)
+  end
 
   def with_verbosity(level)
     old_level = self.verbosity
@@ -97,14 +101,6 @@ class Mrsk::Commander
     SSHKit.config.output_verbosity = old_level
   end
 
-
-  # Test-induced damage!
-  def reset
-    @config = nil
-    @app = @builder = @traefik = @registry = @prune = @auditor = nil
-    @verbosity = :info
-  end
-
   private
     # Lazy setup of SSHKit
     def configure_sshkit_with(config)

lib/mrsk/commands/base.rb

@@ -41,6 +41,10 @@ module Mrsk::Commands
         combine *commands, by: ">>"
       end
 
+      def write(*commands)
+        combine *commands, by: ">"
+      end
+
       def xargs(command)
         [ :xargs, command ].flatten
       end

lib/mrsk/commands/lock.rb

@@ -0,0 +1,63 @@
+require "active_support/duration"
+require "active_support/core_ext/numeric/time"
+
+class Mrsk::Commands::Lock < Mrsk::Commands::Base
+  def acquire(message, version)
+    combine \
+      [:mkdir, lock_dir],
+      write_lock_details(message, version)
+  end
+
+  def release
+    combine \
+      [:rm, lock_details_file],
+      [:rm, "-r", lock_dir]
+  end
+
+  def status
+    combine \
+      stat_lock_dir,
+      read_lock_details
+  end
+
+  private
+    def write_lock_details(message, version)
+      write \
+        [:echo, "\"#{Base64.encode64(lock_details(message, version))}\""],
+        lock_details_file
+    end
+
+    def read_lock_details
+      pipe \
+        [:cat, lock_details_file],
+        [:base64, "-d"]
+    end
+
+    def stat_lock_dir
+      write \
+        [:stat, lock_dir],
+        "/dev/null"
+    end
+
+    def lock_dir
+      :mrsk_lock
+    end
+
+    def lock_details_file
+      [lock_dir, :details].join("/")
+    end
+
+    def lock_details(message, version)
+      <<~DETAILS.strip
+        Locked by: #{locked_by} at #{Time.now.gmtime}
+        Version: #{version}
+        Message: #{message}
+      DETAILS
+    end
+
+    def locked_by
+      `git config user.name`.strip
+    rescue Errno::ENOENT
+      "Unknown"
+    end
+end

test/cli/build_test.rb

@@ -15,8 +15,9 @@ class CliBuildTest < CliTestCase
   end
 
   test "push without builder" do
-    Mrsk::Cli::Build.any_instance.stubs(:create).returns(true)
+    stub_locking
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |arg| arg == :docker }
       .raises(SSHKit::Command::Failed.new("no builder"))
       .then
       .returns(true)
@@ -40,7 +41,9 @@ class CliBuildTest < CliTestCase
   end
 
   test "create with error" do
+    stub_locking
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |arg| arg == :docker }
       .raises(SSHKit::Command::Failed.new("stderr=error"))
 
     run_command("create").tap do |output|
@@ -69,4 +72,11 @@ class CliBuildTest < CliTestCase
     def run_command(*command)
       stdouted { Mrsk::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
     end
+
+    def stub_locking
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock }
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" }
+    end
 end

test/cli/cli_test_case.rb

@@ -8,13 +8,14 @@ class CliTestCase < ActiveSupport::TestCase
     ENV["VERSION"]             = "999"
     ENV["RAILS_MASTER_KEY"]    = "123"
     ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
+    Object.send(:remove_const, :MRSK)
+    Object.const_set(:MRSK, Mrsk::Commander.new)
   end
 
   teardown do
     ENV.delete("RAILS_MASTER_KEY")
     ENV.delete("MYSQL_ROOT_PASSWORD")
     ENV.delete("VERSION")
-    MRSK.reset
   end
 
   private

test/cli/lock_test.rb

@@ -0,0 +1,20 @@
+require_relative "cli_test_case"
+
+class CliLockTest < CliTestCase
+  test "status" do
+    run_command("status") do |output|
+      assert_match "stat lock", output
+    end
+  end
+
+  test "release" do
+    run_command("release") do |output|
+      assert_match "rm -rf lock", output
+    end
+  end
+
+  private
+    def run_command(*command)
+      stdouted { Mrsk::Cli::Lock.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+    end
+end

test/cli/main_test.rb

@@ -42,12 +42,14 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
 
     run_command("deploy", "--skip_push").tap do |output|
+      assert_match /Acquiring the deploy lock/, output
       assert_match /Ensure curl and Docker are installed/, output
       assert_match /Log into image registry/, output
       assert_match /Pull app image/, output
       assert_match /Ensure Traefik is running/, output
       assert_match /Ensure app can pass healthcheck/, output
       assert_match /Prune old containers and images/, output
+      assert_match /Releasing the deploy lock/, output
     end
   end
 

test/commands/lock_test.rb

@@ -0,0 +1,33 @@
+require "test_helper"
+
+class CommandsLockTest < ActiveSupport::TestCase
+  setup do
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
+    }
+  end
+
+  test "status" do
+    assert_equal \
+      "stat mrsk_lock > /dev/null && cat mrsk_lock/details | base64 -d",
+      new_command.status.join(" ")
+  end
+
+  test "acquire" do
+    assert_match \
+      /mkdir mrsk_lock && echo ".*" > mrsk_lock\/details/m,
+      new_command.acquire("Hello", "123").join(" ")
+  end
+
+  test "release" do
+    assert_match \
+      "rm mrsk_lock/details && rm -r mrsk_lock",
+      new_command.release.join(" ")
+  end
+
+  private
+    def new_command
+      Mrsk::Commands::Lock.new(Mrsk::Configuration.new(@config, version: "123"))
+    end
+end