Deploy locks

Add a deploy lock for commands that are unsafe to run concurrently.

The lock is taken by creating a `mrsk_lock` directory on the primary
host. Details of who took the lock are added to a details file in that
directory.

Additional CLI commands have been added to manual release and acquire
the lock and to check its status.

```
Commands:
  mrsk lock acquire -m, --message=MESSAGE  # Acquire the deploy lock
  mrsk lock help [COMMAND]                 # Describe subcommands or one specific subcommand
  mrsk lock release                        # Release the deploy lock
  mrsk lock status                         # Report lock status

Options:
  -v, [--verbose], [--no-verbose]                # Detailed logging
  -q, [--quiet], [--no-quiet]                    # Minimal logging
      [--version=VERSION]                        # Run commands against a specific app version
  -p, [--primary], [--no-primary]                # Run commands only on primary host instead of all
  -h, [--hosts=HOSTS]                            # Run commands on these hosts instead of all (separate by comma)
  -r, [--roles=ROLES]                            # Run commands on these roles instead of all (separate by comma)
  -c, [--config-file=CONFIG_FILE]                # Path to config file
                                                 # Default: config/deploy.yml
  -d, [--destination=DESTINATION]                # Specify destination to be used for config file (staging -> deploy.staging.yml)
  -B, [--skip-broadcast], [--no-skip-broadcast]  # Skip audit broadcasts
```

If we add support for running multiple deployments on a single server
we'll need to extend the locking to lock per deployment.

test/cli/build_test.rb

@@ -15,8 +15,9 @@ class CliBuildTest < CliTestCase
   end
 
   test "push without builder" do
-    Mrsk::Cli::Build.any_instance.stubs(:create).returns(true)
+    stub_locking
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |arg| arg == :docker }
       .raises(SSHKit::Command::Failed.new("no builder"))
       .then
       .returns(true)
@@ -40,7 +41,9 @@ class CliBuildTest < CliTestCase
   end
 
   test "create with error" do
+    stub_locking
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+      .with { |arg| arg == :docker }
       .raises(SSHKit::Command::Failed.new("stderr=error"))
 
     run_command("create").tap do |output|
@@ -69,4 +72,11 @@ class CliBuildTest < CliTestCase
     def run_command(*command)
       stdouted { Mrsk::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
     end
+
+    def stub_locking
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock }
+      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
+        .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" }
+    end
 end

test/cli/cli_test_case.rb

@@ -8,13 +8,14 @@ class CliTestCase < ActiveSupport::TestCase
     ENV["VERSION"]             = "999"
     ENV["RAILS_MASTER_KEY"]    = "123"
     ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
+    Object.send(:remove_const, :MRSK)
+    Object.const_set(:MRSK, Mrsk::Commander.new)
   end
 
   teardown do
     ENV.delete("RAILS_MASTER_KEY")
     ENV.delete("MYSQL_ROOT_PASSWORD")
     ENV.delete("VERSION")
-    MRSK.reset
   end
 
   private

test/cli/lock_test.rb

@@ -0,0 +1,20 @@
+require_relative "cli_test_case"
+
+class CliLockTest < CliTestCase
+  test "status" do
+    run_command("status") do |output|
+      assert_match "stat lock", output
+    end
+  end
+
+  test "release" do
+    run_command("release") do |output|
+      assert_match "rm -rf lock", output
+    end
+  end
+
+  private
+    def run_command(*command)
+      stdouted { Mrsk::Cli::Lock.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+    end
+end

test/cli/main_test.rb

@@ -42,12 +42,14 @@ class CliMainTest < CliTestCase
     Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options)
 
     run_command("deploy", "--skip_push").tap do |output|
+      assert_match /Acquiring the deploy lock/, output
       assert_match /Ensure curl and Docker are installed/, output
       assert_match /Log into image registry/, output
       assert_match /Pull app image/, output
       assert_match /Ensure Traefik is running/, output
       assert_match /Ensure app can pass healthcheck/, output
       assert_match /Prune old containers and images/, output
+      assert_match /Releasing the deploy lock/, output
     end
   end
 

test/commands/lock_test.rb

@@ -0,0 +1,33 @@
+require "test_helper"
+
+class CommandsLockTest < ActiveSupport::TestCase
+  setup do
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
+    }
+  end
+
+  test "status" do
+    assert_equal \
+      "stat mrsk_lock > /dev/null && cat mrsk_lock/details | base64 -d",
+      new_command.status.join(" ")
+  end
+
+  test "acquire" do
+    assert_match \
+      /mkdir mrsk_lock && echo ".*" > mrsk_lock\/details/m,
+      new_command.acquire("Hello", "123").join(" ")
+  end
+
+  test "release" do
+    assert_match \
+      "rm mrsk_lock/details && rm -r mrsk_lock",
+      new_command.release.join(" ")
+  end
+
+  private
+    def new_command
+      Mrsk::Commands::Lock.new(Mrsk::Configuration.new(@config, version: "123"))
+    end
+end