Merge pull request #222 from basecamp/deploy-groups

Allow booting containers in groups for rolling restarts

README.md

@@ -855,6 +855,24 @@ mrsk lock acquire -m "Doing maintanence"
 mrsk lock release
 ```
 
+## Rolling deployments
+
+When deploying to large numbers of hosts, you might prefer not to restart your services on every host at the same time.
+
+MRSK's default is to boot new containers on all hosts in parallel. But you can control this by configuring `boot/limit` and `boot/wait` as options:
+
+```yaml
+service: myservice
+
+boot:
+  limit: 10 # Can also specify as a percentage of total hosts, such as "25%"
+  wait: 2
+```
+
+When `limit` is specified, containers will be booted on, at most, `limit` hosts at once. MRSK will pause for `wait` seconds between batches.
+
+These settings only apply when booting containers (using `mrsk deploy`, or `mrsk app boot`). For other commands, MRSK continues to run commands in parallel across all hosts.
+
 ## Stage of development
 
 This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com).

lib/mrsk/cli/app.rb

@@ -11,7 +11,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
           execute *MRSK.app.tag_current_as_latest
         end
 
-        on(MRSK.hosts) do |host|
+        on(MRSK.hosts, **MRSK.boot_strategy) do |host|
           roles = MRSK.roles_on(host)
 
           roles.each do |role|

lib/mrsk/commander.rb

@@ -51,6 +51,14 @@ class Mrsk::Commander
     end
   end
 
+  def boot_strategy
+    if config.boot.limit.present?
+      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
+    else
+      {}
+    end
+  end
+
   def roles_on(host)
     roles.select { |role| role.hosts.include?(host.to_s) }.map(&:name)
   end

lib/mrsk/configuration.rb

@@ -87,6 +87,10 @@ class Mrsk::Configuration
     roles.select(&:running_traefik?).flat_map(&:hosts).uniq
   end
 
+  def boot
+    Mrsk::Configuration::Boot.new(config: self)
+  end
+
 
   def repository
     [ raw_config.registry["server"], image ].compact.join("/")

lib/mrsk/configuration/boot.rb

@@ -0,0 +1,20 @@
+class Mrsk::Configuration::Boot
+  def initialize(config:)
+    @options = config.raw_config.boot || {}
+    @host_count = config.all_hosts.count
+  end
+
+  def limit
+    limit = @options["limit"]
+
+    if limit.to_s.end_with?("%")
+      @host_count * limit.to_i / 100
+    else
+      limit
+    end
+  end
+
+  def wait
+    @options["wait"]
+  end
+end

test/cli/app_test.rb

@@ -40,6 +40,16 @@ class CliAppTest < CliTestCase
     Thread.report_on_exception = true
   end
 
+  test "boot uses group strategy when specified" do
+    Mrsk::Cli::App.any_instance.stubs(:on).with("1.1.1.1").twice # acquire & release lock
+    Mrsk::Cli::App.any_instance.stubs(:on).with([ "1.1.1.1" ]) # tag container
+
+    # Strategy is used when booting the containers
+    Mrsk::Cli::App.any_instance.expects(:on).with([ "1.1.1.1" ], in: :groups, limit: 3, wait: 2).with_block_given
+
+    run_command("boot", config: :with_boot_strategy)
+  end
+
   test "start" do
     run_command("start").tap do |output|
       assert_match "docker start app-web-999", output
@@ -158,7 +168,7 @@ class CliAppTest < CliTestCase
   end
 
   private
-    def run_command(*command)
+    def run_command(*command, config: :with_accessories)
-      stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1"]) }
+      stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1"]) }
     end
 end

test/commander_test.rb

@@ -2,9 +2,7 @@ require "test_helper"
 
 class CommanderTest < ActiveSupport::TestCase
   setup do
-    @mrsk = Mrsk::Commander.new.tap do |mrsk|
+    configure_with(:deploy_with_roles)
-      mrsk.configure config_file: Pathname.new(File.expand_path("fixtures/deploy_with_roles.yml", __dir__))
-    end
   end
 
   test "lazy configuration" do
@@ -55,4 +53,27 @@ class CommanderTest < ActiveSupport::TestCase
     assert_equal [ "web" ], @mrsk.roles_on("1.1.1.1")
     assert_equal [ "workers" ], @mrsk.roles_on("1.1.1.3")
   end
+
+  test "default group strategy" do
+    assert_empty @mrsk.boot_strategy
+  end
+
+  test "specific limit group strategy" do
+    configure_with(:deploy_with_boot_strategy)
+
+    assert_equal({ in: :groups, limit: 3, wait: 2 }, @mrsk.boot_strategy)
+  end
+
+  test "percentage-based group strategy" do
+    configure_with(:deploy_with_precentage_boot_strategy)
+
+    assert_equal({ in: :groups, limit: 1, wait: 2 }, @mrsk.boot_strategy)
+  end
+
+  private
+    def configure_with(variant)
+      @mrsk = Mrsk::Commander.new.tap do |mrsk|
+        mrsk.configure config_file: Pathname.new(File.expand_path("fixtures/#{variant}.yml", __dir__))
+      end
+    end
 end

test/fixtures/deploy_with_boot_strategy.yml

@@ -0,0 +1,17 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+
+registry:
+  username: user
+  password: pw
+
+boot:
+  limit: 3
+  wait: 2

test/fixtures/deploy_with_precentage_boot_strategy.yml

@@ -0,0 +1,17 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+
+registry:
+  username: user
+  password: pw
+
+boot:
+  limit: 25%
+  wait: 2