diff --git a/README.md b/README.md index 95042cd9..8ae2121c 100644 --- a/README.md +++ b/README.md @@ -855,6 +855,24 @@ mrsk lock acquire -m "Doing maintanence" mrsk lock release ``` +## Rolling deployments + +When deploying to large numbers of hosts, you might prefer not to restart your services on every host at the same time. + +MRSK's default is to boot new containers on all hosts in parallel. But you can control this by configuring `boot/limit` and `boot/wait` as options: + +```yaml +service: myservice + +boot: + limit: 10 # Can also specify as a percentage of total hosts, such as "25%" + wait: 2 +``` + +When `limit` is specified, containers will be booted on, at most, `limit` hosts at once. MRSK will pause for `wait` seconds between batches. + +These settings only apply when booting containers (using `mrsk deploy`, or `mrsk app boot`). For other commands, MRSK continues to run commands in parallel across all hosts. + ## Stage of development This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com). diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb index 6e4fd496..9e783efb 100644 --- a/lib/mrsk/cli/app.rb +++ b/lib/mrsk/cli/app.rb @@ -11,7 +11,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base execute *MRSK.app.tag_current_as_latest end - on(MRSK.hosts) do |host| + on(MRSK.hosts, **MRSK.boot_strategy) do |host| roles = MRSK.roles_on(host) roles.each do |role| diff --git a/lib/mrsk/commander.rb b/lib/mrsk/commander.rb index 44c96a2a..217918e6 100644 --- a/lib/mrsk/commander.rb +++ b/lib/mrsk/commander.rb @@ -51,6 +51,14 @@ class Mrsk::Commander end end + def boot_strategy + if config.boot.limit.present? + { in: :groups, limit: config.boot.limit, wait: config.boot.wait } + else + {} + end + end + def roles_on(host) roles.select { |role| role.hosts.include?(host.to_s) }.map(&:name) end diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index a2c2ecf1..4d2d010a 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -87,6 +87,10 @@ class Mrsk::Configuration roles.select(&:running_traefik?).flat_map(&:hosts).uniq end + def boot + Mrsk::Configuration::Boot.new(config: self) + end + def repository [ raw_config.registry["server"], image ].compact.join("/") diff --git a/lib/mrsk/configuration/boot.rb b/lib/mrsk/configuration/boot.rb new file mode 100644 index 00000000..1332398a --- /dev/null +++ b/lib/mrsk/configuration/boot.rb @@ -0,0 +1,20 @@ +class Mrsk::Configuration::Boot + def initialize(config:) + @options = config.raw_config.boot || {} + @host_count = config.all_hosts.count + end + + def limit + limit = @options["limit"] + + if limit.to_s.end_with?("%") + @host_count * limit.to_i / 100 + else + limit + end + end + + def wait + @options["wait"] + end +end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 54292d65..977f2900 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -40,6 +40,16 @@ class CliAppTest < CliTestCase Thread.report_on_exception = true end + test "boot uses group strategy when specified" do + Mrsk::Cli::App.any_instance.stubs(:on).with("1.1.1.1").twice # acquire & release lock + Mrsk::Cli::App.any_instance.stubs(:on).with([ "1.1.1.1" ]) # tag container + + # Strategy is used when booting the containers + Mrsk::Cli::App.any_instance.expects(:on).with([ "1.1.1.1" ], in: :groups, limit: 3, wait: 2).with_block_given + + run_command("boot", config: :with_boot_strategy) + end + test "start" do run_command("start").tap do |output| assert_match "docker start app-web-999", output @@ -158,7 +168,7 @@ class CliAppTest < CliTestCase end private - def run_command(*command) - stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1"]) } + def run_command(*command, config: :with_accessories) + stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1"]) } end end diff --git a/test/commander_test.rb b/test/commander_test.rb index 163feaf7..25dfabdd 100644 --- a/test/commander_test.rb +++ b/test/commander_test.rb @@ -2,9 +2,7 @@ require "test_helper" class CommanderTest < ActiveSupport::TestCase setup do - @mrsk = Mrsk::Commander.new.tap do |mrsk| - mrsk.configure config_file: Pathname.new(File.expand_path("fixtures/deploy_with_roles.yml", __dir__)) - end + configure_with(:deploy_with_roles) end test "lazy configuration" do @@ -55,4 +53,27 @@ class CommanderTest < ActiveSupport::TestCase assert_equal [ "web" ], @mrsk.roles_on("1.1.1.1") assert_equal [ "workers" ], @mrsk.roles_on("1.1.1.3") end + + test "default group strategy" do + assert_empty @mrsk.boot_strategy + end + + test "specific limit group strategy" do + configure_with(:deploy_with_boot_strategy) + + assert_equal({ in: :groups, limit: 3, wait: 2 }, @mrsk.boot_strategy) + end + + test "percentage-based group strategy" do + configure_with(:deploy_with_precentage_boot_strategy) + + assert_equal({ in: :groups, limit: 1, wait: 2 }, @mrsk.boot_strategy) + end + + private + def configure_with(variant) + @mrsk = Mrsk::Commander.new.tap do |mrsk| + mrsk.configure config_file: Pathname.new(File.expand_path("fixtures/#{variant}.yml", __dir__)) + end + end end diff --git a/test/fixtures/deploy_with_boot_strategy.yml b/test/fixtures/deploy_with_boot_strategy.yml new file mode 100644 index 00000000..7691eb2e --- /dev/null +++ b/test/fixtures/deploy_with_boot_strategy.yml @@ -0,0 +1,17 @@ +service: app +image: dhh/app +servers: + web: + - "1.1.1.1" + - "1.1.1.2" + workers: + - "1.1.1.3" + - "1.1.1.4" + +registry: + username: user + password: pw + +boot: + limit: 3 + wait: 2 diff --git a/test/fixtures/deploy_with_precentage_boot_strategy.yml b/test/fixtures/deploy_with_precentage_boot_strategy.yml new file mode 100644 index 00000000..eb68a52f --- /dev/null +++ b/test/fixtures/deploy_with_precentage_boot_strategy.yml @@ -0,0 +1,17 @@ +service: app +image: dhh/app +servers: + web: + - "1.1.1.1" + - "1.1.1.2" + workers: + - "1.1.1.3" + - "1.1.1.4" + +registry: + username: user + password: pw + +boot: + limit: 25% + wait: 2