Merge branch 'main' into rolling-traefik-restarts

* main:
  Removed not needed MRSK.traefik.run command in Traefil reboot
  Updated README with locking directory name
  Include service name to lock details
  Configurable SSH log levels
  Add registry container output to debug
  Minor tweaks to hooks section in readme
  Update README.md
  Updated README.md to make setup examples consistent
  Login to the registry proactively before stoping Accessory and Traefik

README.md

@@ -123,6 +123,8 @@ This template can safely be checked into git. Then everyone deploying the app ca
 
 If you need separate env variables for different destinations, you can set them with `.env.destination.erb` for the template, which will generate `.env.staging` when run with `mrsk envify -d staging`.
 
+Note: If you utilize biometrics with 1Password you can remove the `session_token` related parts in the example and just call `op read op://Vault/Docker Hub/password -n`.
+
 #### Bitwarden as a secret store
 
 If you are using open source secret store like bitwarden, you can create `.env.erb` as a template which looks up the secrets.
@@ -206,13 +208,13 @@ ssh:
   user: app
 ```
 
-If you are using non-root user, you need to bootstrap your servers manually, before using them with MRSK. On Ubuntu, you'd do:
+If you are using non-root user (`app` as above example), you need to bootstrap your servers manually, before using them with MRSK. On Ubuntu, you'd do:
 
 ```bash
 sudo apt update
 sudo apt upgrade -y
 sudo apt install -y docker.io curl git
-sudo usermod -a -G docker ubuntu
+sudo usermod -a -G docker app
 ```
 
 ### Using a proxy SSH host
@@ -238,6 +240,15 @@ ssh:
   proxy_command: aws ssm start-session --target %h --document-name AWS-StartSSHSession --parameters 'portNumber=%p' --region=us-east-1 ## ssh via aws ssm
 ```
 
+### Configuring the SSH log level
+
+```yaml
+ssh:
+  log_level: debug
+```
+
+Valid levels are `debug`, `info`, `warn`, `error` and `fatal` (default).
+
 ### Using env variables
 
 You can inject env variables into the app containers using `env`:
@@ -866,7 +877,7 @@ If you wish to remove the entire application, including Traefik, containers, ima
 
 ## Locking
 
-Commands that are unsafe to run concurrently will take a deploy lock while they run. The lock is the `mrsk_lock` directory on the primary server.
+Commands that are unsafe to run concurrently will take a deploy lock while they run. The lock is the `mrsk_lock-<service>` directory on the primary server.
 
 You can check the lock status with:
 
@@ -922,8 +933,8 @@ firing a JSON webhook. These variables include:
 - `MRSK_RECORDED_AT` - UTC timestamp in ISO 8601 format, e.g. `2023-04-14T17:07:31Z`
 - `MRSK_PERFORMER` - the local user performing the command (from `whoami`)
 - `MRSK_SERVICE_VERSION` - an abbreviated service and version for use in messages, e.g. app@150b24f
-- `MRSK_VERSION` - an full version being deployed
+- `MRSK_VERSION` - the full version being deployed
-- `MRSK_HOSTS` - a comma separated list of the hosts targeted by the command
+- `MRSK_HOSTS` - a comma-separated list of the hosts targeted by the command
 - `MRSK_COMMAND` - The command we are running
 - `MRSK_SUBCOMMAND` - optional: The subcommand we are running
 - `MRSK_DESTINATION` - optional: destination, e.g. "staging"
@@ -940,9 +951,8 @@ Used for pre-build checks - e.g. there are no uncommitted changes or that CI has
 3. pre-deploy
 For final checks before deploying, e.g. checking CI completed
 
-3. post-deploy - run after a deploy, redeploy or rollback
+3. post-deploy - run after a deploy, redeploy or rollback.
-
+This hook is also passed a `MRSK_RUNTIME` env variable set to the total seconds the deploy took.
-This hook is also passed a `MRSK_RUNTIME` env variable.
 
 This could be used to broadcast a deployment message, or register the new version with an APM.
 
@@ -953,7 +963,7 @@ The command could look something like:
 curl -q -d content="[My App] ${MRSK_PERFORMER} Rolled back to version ${MRSK_VERSION}" https://3.basecamp.com/XXXXX/integrations/XXXXX/buckets/XXXXX/chats/XXXXX/lines
 ```
 
-That'll post a line like follows to a preconfigured chatbot in Basecamp:
+That'll post a line like the following to a preconfigured chatbot in Basecamp:
 
 ```
 [My App] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de

lib/mrsk/cli/accessory.rb

@@ -1,6 +1,6 @@
 class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   desc "boot [NAME]", "Boot new accessory service on host (use NAME=all to boot all accessories)"
-  def boot(name)
+  def boot(name, login: true)
     mutating do
       if name == "all"
         MRSK.accessory_names.each { |accessory_name| boot(accessory_name) }
@@ -10,7 +10,7 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
           upload(name)
 
           on(accessory.hosts) do
-            execute *MRSK.registry.login
+            execute *MRSK.registry.login if login
             execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug
             execute *accessory.run
           end
@@ -53,9 +53,13 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   def reboot(name)
     mutating do
       with_accessory(name) do |accessory|
+        on(accessory.hosts) do
+          execute *MRSK.registry.login
+        end
+
         stop(name)
         remove_container(name)
-        boot(name)
+        boot(name, login: false)
       end
     end
   end

lib/mrsk/cli/traefik.rb

@@ -15,9 +15,9 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
     mutating do
       on(MRSK.traefik_hosts, in: options[:rolling] ? :sequence : :parallel) do
         execute *MRSK.auditor.record("Rebooted traefik"), verbosity: :debug
+        execute *MRSK.registry.login
         execute *MRSK.traefik.stop, raise_on_non_zero_exit: false
         execute *MRSK.traefik.remove_container
-        execute *MRSK.registry.login
         execute *MRSK.traefik.run, raise_on_non_zero_exit: false
       end
     end

lib/mrsk/commands/lock.rb

@@ -40,7 +40,7 @@ class Mrsk::Commands::Lock < Mrsk::Commands::Base
     end
 
     def lock_dir
-      :mrsk_lock
+      "mrsk_lock-#{config.service}"
     end
 
     def lock_details_file

lib/mrsk/configuration.rb

@@ -153,7 +153,15 @@ class Mrsk::Configuration
   end
 
   def ssh_options
-    { user: ssh_user, proxy: ssh_proxy, auth_methods: [ "publickey" ] }.compact
+    { user: ssh_user, proxy: ssh_proxy, auth_methods: [ "publickey" ], logger: ssh_logger }.compact
+  end
+
+  def ssh_logger
+    @ssh_logger ||= ::Logger.new(STDERR).tap { |logger| logger.level = ssh_log_level }
+  end
+
+  def ssh_log_level
+    (raw_config.ssh && raw_config.ssh["log_level"]) || ::Logger::FATAL
   end
 
 
@@ -185,7 +193,8 @@ class Mrsk::Configuration
       service_with_version: service_with_version,
       env_args: env_args,
       volume_args: volume_args,
-      ssh_options: ssh_options,
+      ssh_options: ssh_options.except(:logger),
+      ssh_log_level: ssh_log_level,
       builder: builder.to_h,
       accessories: raw_config.accessories,
       logging: logging_args,

test/cli/accessory_test.rb

@@ -40,9 +40,10 @@ class CliAccessoryTest < CliTestCase
   end
 
   test "reboot" do
+    Mrsk::Commands::Registry.any_instance.expects(:login)
     Mrsk::Cli::Accessory.any_instance.expects(:stop).with("mysql")
     Mrsk::Cli::Accessory.any_instance.expects(:remove_container).with("mysql")
-    Mrsk::Cli::Accessory.any_instance.expects(:boot).with("mysql")
+    Mrsk::Cli::Accessory.any_instance.expects(:boot).with("mysql", login: false)
 
     run_command("reboot", "mysql")
   end

test/cli/cli_test_case.rb

@@ -29,9 +29,9 @@ class CliTestCase < ActiveSupport::TestCase
 
     def stub_locking
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock }
+        .with { |arg1, arg2| arg1 == :mkdir && arg2 == "mrsk_lock-app" }
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" }
+        .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock-app/details" }
     end
 
     def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: nil)

test/cli/main_test.rb

@@ -63,11 +63,11 @@ class CliMainTest < CliTestCase
     Thread.report_on_exception = false
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*arg| arg[0..1] == [:mkdir, :mrsk_lock] }
+      .with { |*arg| arg[0..1] == [:mkdir, 'mrsk_lock-app'] }
-      .raises(RuntimeError, "mkdir: cannot create directory ‘mrsk_lock’: File exists")
+      .raises(RuntimeError, "mkdir: cannot create directory ‘mrsk_lock-app’: File exists")
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_debug)
-      .with(:stat, :mrsk_lock, ">", "/dev/null", "&&", :cat, "mrsk_lock/details", "|", :base64, "-d")
+      .with(:stat, 'mrsk_lock-app', ">", "/dev/null", "&&", :cat, "mrsk_lock-app/details", "|", :base64, "-d")
 
     assert_raises(Mrsk::Cli::LockError) do
       run_command("deploy")
@@ -78,7 +78,7 @@ class CliMainTest < CliTestCase
     Thread.report_on_exception = false
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-      .with { |*arg| arg[0..1] == [:mkdir, :mrsk_lock] }
+      .with { |*arg| arg[0..1] == [:mkdir, 'mrsk_lock-app'] }
       .raises(SocketError, "getaddrinfo: nodename nor servname provided, or not known")
 
     assert_raises(SSHKit::Runner::ExecuteError) do

test/cli/traefik_test.rb

@@ -9,17 +9,18 @@ class CliTraefikTest < CliTestCase
   end
 
   test "reboot" do
+    Mrsk::Commands::Registry.any_instance.expects(:login).twice
+
     run_command("reboot").tap do |output|
       assert_match "docker container stop traefik", output
       assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik", output
-      assert_match "docker login", output
       assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" #{Mrsk::Commands::Traefik::DEFAULT_IMAGE} --providers.docker --log.level=\"DEBUG\"", output
     end
   end
 
   test "reboot --rolling" do
     run_command("reboot", "--rolling").tap do |output|
-      assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.1", output.lines[2]
+      assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.1", output.lines[3]
     end
   end
 

test/commands/lock_test.rb

@@ -10,19 +10,19 @@ class CommandsLockTest < ActiveSupport::TestCase
 
   test "status" do
     assert_equal \
-      "stat mrsk_lock > /dev/null && cat mrsk_lock/details | base64 -d",
+      "stat mrsk_lock-app > /dev/null && cat mrsk_lock-app/details | base64 -d",
       new_command.status.join(" ")
   end
 
   test "acquire" do
     assert_match \
-      /mkdir mrsk_lock && echo ".*" > mrsk_lock\/details/m,
+      /mkdir mrsk_lock-app && echo ".*" > mrsk_lock-app\/details/m,
       new_command.acquire("Hello", "123").join(" ")
   end
 
   test "release" do
     assert_match \
-      "rm mrsk_lock/details && rm -r mrsk_lock",
+      "rm mrsk_lock-app/details && rm -r mrsk_lock-app",
       new_command.release.join(" ")
   end
 

test/configuration_test.rb

@@ -211,17 +211,21 @@ class ConfigurationTest < ActiveSupport::TestCase
     assert_equal "root", @config.ssh_options[:user]
 
     config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "user" => "app" }) })
-    assert_equal "app", @config.ssh_options[:user]
+    assert_equal "app", config.ssh_options[:user]
+    assert_equal 4, config.ssh_options[:logger].level
+
+    config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "log_level" => "debug" }) })
+    assert_equal 0, config.ssh_options[:logger].level
   end
 
   test "ssh options with proxy host" do
     config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "proxy" => "1.2.3.4" }) })
-    assert_equal "root@1.2.3.4", @config.ssh_options[:proxy].jump_proxies
+    assert_equal "root@1.2.3.4", config.ssh_options[:proxy].jump_proxies
   end
 
   test "ssh options with proxy host and user" do
     config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "proxy" => "app@1.2.3.4" }) })
-    assert_equal "app@1.2.3.4", @config.ssh_options[:proxy].jump_proxies
+    assert_equal "app@1.2.3.4", config.ssh_options[:proxy].jump_proxies
   end
 
   test "volume_args" do
@@ -266,7 +270,22 @@ class ConfigurationTest < ActiveSupport::TestCase
   end
 
   test "to_h" do
-    assert_equal({ :roles=>["web"], :hosts=>["1.1.1.1", "1.1.1.2"], :primary_host=>"1.1.1.1", :version=>"missing", :repository=>"dhh/app", :absolute_image=>"dhh/app:missing", :service_with_version=>"app-missing", :env_args=>["-e", "REDIS_URL=\"redis://x/y\""], :ssh_options=>{:user=>"root", :auth_methods=>["publickey"]}, :volume_args=>["--volume", "/local/path:/container/path"], :builder=>{}, :logging=>["--log-opt", "max-size=\"10m\""], :healthcheck=>{"path"=>"/up", "port"=>3000, "max_attempts" => 7 }}, @config.to_h)
+    assert_equal({
+      :roles=>["web"],
+      :hosts=>["1.1.1.1", "1.1.1.2"],
+      :primary_host=>"1.1.1.1",
+      :version=>"missing",
+      :repository=>"dhh/app",
+      :absolute_image=>"dhh/app:missing",
+      :service_with_version=>"app-missing",
+      :env_args=>["-e", "REDIS_URL=\"redis://x/y\""],
+      :ssh_options=>{:user=>"root", :auth_methods=>["publickey"]},
+      :ssh_log_level=>4,
+      :volume_args=>["--volume", "/local/path:/container/path"],
+      :builder=>{},
+      :logging=>["--log-opt", "max-size=\"10m\""],
+      :healthcheck=>{"path"=>"/up", "port"=>3000, "max_attempts" => 7 }
+    }, @config.to_h)
   end
 
   test "min version is lower" do

test/integration/integration_test.rb

@@ -11,7 +11,7 @@ class IntegrationTest < ActiveSupport::TestCase
 
   teardown do
     unless passed?
-      [:deployer, :vm1, :vm2, :shared, :load_balancer].each do |container|
+      [:deployer, :vm1, :vm2, :shared, :load_balancer, :registry].each do |container|
         puts
         puts "Logs for #{container}:"
         docker_compose :logs, container