From f530009a6ee52c58ab165a4da14a39d93591b495 Mon Sep 17 00:00:00 2001
From: Kevin McConnell <kevin@37signals.com>
Date: Thu, 13 Apr 2023 12:43:19 +0100
Subject: [PATCH] Allow performing boot & start operations in groups

Adds top-level configuration options for `group_limit` and `group_wait`.
When a `group_limit` is present, we'll perform app boot & start
operations on no more than `group_limit` hosts at a time, optionally
sleeping for `group_wait` seconds after each batch.

We currently only do this batching on boot & start operations (including
when they are part of a deployment). Other commands, like `app stop` or
`app details` still work on all hosts in parallel.
---
 README.md                                    | 17 +++++++++++++++++
 lib/mrsk/cli/app.rb                          |  4 ++--
 lib/mrsk/configuration.rb                    | 17 +++++++++++++++++
 test/cli/app_test.rb                         | 14 ++++++++++++--
 test/fixtures/deploy_with_group_strategy.yml | 16 ++++++++++++++++
 5 files changed, 64 insertions(+), 4 deletions(-)
 create mode 100644 test/fixtures/deploy_with_group_strategy.yml

diff --git a/README.md b/README.md
index 23fc8ad6..d972b298 100644
--- a/README.md
+++ b/README.md
@@ -831,6 +831,23 @@ mrsk lock acquire -m "Doing maintanence"
 mrsk lock release
 ```
 
+## Gradual restarts
+
+When deploying to large numbers of hosts, you might prefer not to restart your services on every host at the same time.
+
+MRSK's default is to start new containers on all hosts in parallel. But you can control this by configuring `group_limit` and `group_wait`.
+
+```yaml
+service: myservice
+
+group_limit: 10
+group_wait: 30
+```
+
+When `group_limit` is specified, containers will be started on, at most, `group_limit` hosts at once. MRSK will pause for `group_wait` seconds between batches.
+
+These settings only apply when starting containers (using `mrsk deploy`, `mrsk app boot` or `mrsk app start`). For other commands, MRSK continues to run commands in parallel across all hosts.
+
 ## Stage of development
 
 This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com).
diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
index 882c80c8..b6c9e1b8 100644
--- a/lib/mrsk/cli/app.rb
+++ b/lib/mrsk/cli/app.rb
@@ -13,7 +13,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
           execute *MRSK.app.tag_current_as_latest
         end
 
-        on(MRSK.hosts) do |host|
+        on(MRSK.hosts, **MRSK.config.group_strategy) do |host|
           roles = MRSK.roles_on(host)
 
           roles.each do |role|
@@ -39,7 +39,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   desc "start", "Start existing app container on servers"
   def start
     with_lock do
-      on(MRSK.hosts) do |host|
+      on(MRSK.hosts, **MRSK.config.group_strategy) do |host|
         roles = MRSK.roles_on(host)
 
         roles.each do |role|
diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb
index a2c2ecf1..388bcce4 100644
--- a/lib/mrsk/configuration.rb
+++ b/lib/mrsk/configuration.rb
@@ -153,6 +153,15 @@ class Mrsk::Configuration
   end
 
 
+  def group_strategy
+    if group_limit.present?
+      { in: :groups, limit: group_limit, wait: group_wait }
+    else
+      {}
+    end
+  end
+
+
   def audit_broadcast_cmd
     raw_config.audit_broadcast_cmd
   end
@@ -237,4 +246,12 @@ class Mrsk::Configuration
           raise "Can't use commit hash as version, no git repository found in #{Dir.pwd}"
         end
     end
+
+    def group_limit
+      raw_config.group_limit&.to_i
+    end
+
+    def group_wait
+      raw_config.group_wait&.to_i
+    end
 end
diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb
index 78b166d4..f27c04ea 100644
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -33,6 +33,16 @@ class CliAppTest < CliTestCase
     Thread.report_on_exception = true
   end
 
+  test "boot uses group strategy when specified" do
+    Mrsk::Cli::App.any_instance.stubs(:on).with("1.1.1.1").twice # acquire & release lock
+    Mrsk::Cli::App.any_instance.stubs(:on).with([ "1.1.1.1" ]) # tag container
+
+    # Strategy is used when booting the containers
+    Mrsk::Cli::App.any_instance.expects(:on).with([ "1.1.1.1" ], in: :groups, limit: 3, wait: 30).with_block_given
+
+    run_command("boot", config: :with_group_strategy)
+  end
+
   test "start" do
     run_command("start").tap do |output|
       assert_match "docker start app-web-999", output
@@ -151,7 +161,7 @@ class CliAppTest < CliTestCase
   end
 
   private
-    def run_command(*command)
-      stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1"]) }
+    def run_command(*command, config: :with_accessories)
+      stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1"]) }
     end
 end
diff --git a/test/fixtures/deploy_with_group_strategy.yml b/test/fixtures/deploy_with_group_strategy.yml
new file mode 100644
index 00000000..bc551f8a
--- /dev/null
+++ b/test/fixtures/deploy_with_group_strategy.yml
@@ -0,0 +1,16 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+
+registry:
+  username: user
+  password: pw
+
+group_limit: 3
+group_wait: 30