From f530009a6ee52c58ab165a4da14a39d93591b495 Mon Sep 17 00:00:00 2001 From: Kevin McConnell Date: Thu, 13 Apr 2023 12:43:19 +0100 Subject: [PATCH 1/5] Allow performing boot & start operations in groups Adds top-level configuration options for `group_limit` and `group_wait`. When a `group_limit` is present, we'll perform app boot & start operations on no more than `group_limit` hosts at a time, optionally sleeping for `group_wait` seconds after each batch. We currently only do this batching on boot & start operations (including when they are part of a deployment). Other commands, like `app stop` or `app details` still work on all hosts in parallel. --- README.md | 17 +++++++++++++++++ lib/mrsk/cli/app.rb | 4 ++-- lib/mrsk/configuration.rb | 17 +++++++++++++++++ test/cli/app_test.rb | 14 ++++++++++++-- test/fixtures/deploy_with_group_strategy.yml | 16 ++++++++++++++++ 5 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 test/fixtures/deploy_with_group_strategy.yml diff --git a/README.md b/README.md index 23fc8ad6..d972b298 100644 --- a/README.md +++ b/README.md @@ -831,6 +831,23 @@ mrsk lock acquire -m "Doing maintanence" mrsk lock release ``` +## Gradual restarts + +When deploying to large numbers of hosts, you might prefer not to restart your services on every host at the same time. + +MRSK's default is to start new containers on all hosts in parallel. But you can control this by configuring `group_limit` and `group_wait`. + +```yaml +service: myservice + +group_limit: 10 +group_wait: 30 +``` + +When `group_limit` is specified, containers will be started on, at most, `group_limit` hosts at once. MRSK will pause for `group_wait` seconds between batches. + +These settings only apply when starting containers (using `mrsk deploy`, `mrsk app boot` or `mrsk app start`). For other commands, MRSK continues to run commands in parallel across all hosts. + ## Stage of development This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com). diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb index 882c80c8..b6c9e1b8 100644 --- a/lib/mrsk/cli/app.rb +++ b/lib/mrsk/cli/app.rb @@ -13,7 +13,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base execute *MRSK.app.tag_current_as_latest end - on(MRSK.hosts) do |host| + on(MRSK.hosts, **MRSK.config.group_strategy) do |host| roles = MRSK.roles_on(host) roles.each do |role| @@ -39,7 +39,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base desc "start", "Start existing app container on servers" def start with_lock do - on(MRSK.hosts) do |host| + on(MRSK.hosts, **MRSK.config.group_strategy) do |host| roles = MRSK.roles_on(host) roles.each do |role| diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index a2c2ecf1..388bcce4 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -153,6 +153,15 @@ class Mrsk::Configuration end + def group_strategy + if group_limit.present? + { in: :groups, limit: group_limit, wait: group_wait } + else + {} + end + end + + def audit_broadcast_cmd raw_config.audit_broadcast_cmd end @@ -237,4 +246,12 @@ class Mrsk::Configuration raise "Can't use commit hash as version, no git repository found in #{Dir.pwd}" end end + + def group_limit + raw_config.group_limit&.to_i + end + + def group_wait + raw_config.group_wait&.to_i + end end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 78b166d4..f27c04ea 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -33,6 +33,16 @@ class CliAppTest < CliTestCase Thread.report_on_exception = true end + test "boot uses group strategy when specified" do + Mrsk::Cli::App.any_instance.stubs(:on).with("1.1.1.1").twice # acquire & release lock + Mrsk::Cli::App.any_instance.stubs(:on).with([ "1.1.1.1" ]) # tag container + + # Strategy is used when booting the containers + Mrsk::Cli::App.any_instance.expects(:on).with([ "1.1.1.1" ], in: :groups, limit: 3, wait: 30).with_block_given + + run_command("boot", config: :with_group_strategy) + end + test "start" do run_command("start").tap do |output| assert_match "docker start app-web-999", output @@ -151,7 +161,7 @@ class CliAppTest < CliTestCase end private - def run_command(*command) - stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1"]) } + def run_command(*command, config: :with_accessories) + stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1"]) } end end diff --git a/test/fixtures/deploy_with_group_strategy.yml b/test/fixtures/deploy_with_group_strategy.yml new file mode 100644 index 00000000..bc551f8a --- /dev/null +++ b/test/fixtures/deploy_with_group_strategy.yml @@ -0,0 +1,16 @@ +service: app +image: dhh/app +servers: + web: + - "1.1.1.1" + - "1.1.1.2" + workers: + - "1.1.1.3" + - "1.1.1.4" + +registry: + username: user + password: pw + +group_limit: 3 +group_wait: 30 From 100b72e4b48d4a887746ddf0f8981b1de1096dcf Mon Sep 17 00:00:00 2001 From: Kevin McConnell Date: Fri, 14 Apr 2023 10:41:07 +0100 Subject: [PATCH 2/5] Limit rolling deployment to boot operation --- README.md | 8 ++++---- lib/mrsk/cli/app.rb | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d972b298..e3671f5c 100644 --- a/README.md +++ b/README.md @@ -831,11 +831,11 @@ mrsk lock acquire -m "Doing maintanence" mrsk lock release ``` -## Gradual restarts +## Rolling deployments When deploying to large numbers of hosts, you might prefer not to restart your services on every host at the same time. -MRSK's default is to start new containers on all hosts in parallel. But you can control this by configuring `group_limit` and `group_wait`. +MRSK's default is to boot new containers on all hosts in parallel. But you can control this by configuring `group_limit` and `group_wait`. ```yaml service: myservice @@ -844,9 +844,9 @@ group_limit: 10 group_wait: 30 ``` -When `group_limit` is specified, containers will be started on, at most, `group_limit` hosts at once. MRSK will pause for `group_wait` seconds between batches. +When `group_limit` is specified, containers will be booted on, at most, `group_limit` hosts at once. MRSK will pause for `group_wait` seconds between batches. -These settings only apply when starting containers (using `mrsk deploy`, `mrsk app boot` or `mrsk app start`). For other commands, MRSK continues to run commands in parallel across all hosts. +These settings only apply when booting containers (using `mrsk deploy`, or `mrsk app boot`). For other commands, MRSK continues to run commands in parallel across all hosts. ## Stage of development diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb index b6c9e1b8..735fef88 100644 --- a/lib/mrsk/cli/app.rb +++ b/lib/mrsk/cli/app.rb @@ -39,7 +39,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base desc "start", "Start existing app container on servers" def start with_lock do - on(MRSK.hosts, **MRSK.config.group_strategy) do |host| + on(MRSK.hosts) do |host| roles = MRSK.roles_on(host) roles.each do |role| From a8726be20eda76c68f814c9abfa3854b694b7ecb Mon Sep 17 00:00:00 2001 From: Kevin McConnell Date: Fri, 14 Apr 2023 11:01:25 +0100 Subject: [PATCH 3/5] Move `group_limit` & `group_wait` under `boot` Also make formatting the group strategy the responsibility of the commander. --- README.md | 7 ++++--- lib/mrsk/cli/app.rb | 2 +- lib/mrsk/commander.rb | 8 ++++++++ lib/mrsk/configuration.rb | 21 ++++---------------- lib/mrsk/configuration/boot.rb | 9 +++++++++ test/fixtures/deploy_with_group_strategy.yml | 5 +++-- 6 files changed, 29 insertions(+), 23 deletions(-) create mode 100644 lib/mrsk/configuration/boot.rb diff --git a/README.md b/README.md index e3671f5c..9fcea1ed 100644 --- a/README.md +++ b/README.md @@ -835,13 +835,14 @@ mrsk lock release When deploying to large numbers of hosts, you might prefer not to restart your services on every host at the same time. -MRSK's default is to boot new containers on all hosts in parallel. But you can control this by configuring `group_limit` and `group_wait`. +MRSK's default is to boot new containers on all hosts in parallel. But you can control this by configuring `group_limit` and `group_wait` as boot options: ```yaml service: myservice -group_limit: 10 -group_wait: 30 +boot: + group_limit: 10 + group_wait: 2 ``` When `group_limit` is specified, containers will be booted on, at most, `group_limit` hosts at once. MRSK will pause for `group_wait` seconds between batches. diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb index 735fef88..7f6fbf44 100644 --- a/lib/mrsk/cli/app.rb +++ b/lib/mrsk/cli/app.rb @@ -13,7 +13,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base execute *MRSK.app.tag_current_as_latest end - on(MRSK.hosts, **MRSK.config.group_strategy) do |host| + on(MRSK.hosts, **MRSK.group_strategy) do |host| roles = MRSK.roles_on(host) roles.each do |role| diff --git a/lib/mrsk/commander.rb b/lib/mrsk/commander.rb index 44c96a2a..160b619e 100644 --- a/lib/mrsk/commander.rb +++ b/lib/mrsk/commander.rb @@ -51,6 +51,14 @@ class Mrsk::Commander end end + def group_strategy + if config.boot.group_limit.present? + { in: :groups, limit: config.boot.group_limit, wait: config.boot.group_wait } + else + {} + end + end + def roles_on(host) roles.select { |role| role.hosts.include?(host.to_s) }.map(&:name) end diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index 388bcce4..c3415703 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -87,6 +87,10 @@ class Mrsk::Configuration roles.select(&:running_traefik?).flat_map(&:hosts).uniq end + def boot + Mrsk::Configuration::Boot.new(section: raw_config.boot) + end + def repository [ raw_config.registry["server"], image ].compact.join("/") @@ -153,15 +157,6 @@ class Mrsk::Configuration end - def group_strategy - if group_limit.present? - { in: :groups, limit: group_limit, wait: group_wait } - else - {} - end - end - - def audit_broadcast_cmd raw_config.audit_broadcast_cmd end @@ -246,12 +241,4 @@ class Mrsk::Configuration raise "Can't use commit hash as version, no git repository found in #{Dir.pwd}" end end - - def group_limit - raw_config.group_limit&.to_i - end - - def group_wait - raw_config.group_wait&.to_i - end end diff --git a/lib/mrsk/configuration/boot.rb b/lib/mrsk/configuration/boot.rb new file mode 100644 index 00000000..150f3d5f --- /dev/null +++ b/lib/mrsk/configuration/boot.rb @@ -0,0 +1,9 @@ +class Mrsk::Configuration::Boot + attr_reader :group_wait, :group_limit + + def initialize(section:) + section = section || {} + @group_limit = section["group_limit"] + @group_wait = section["group_wait"] + end +end diff --git a/test/fixtures/deploy_with_group_strategy.yml b/test/fixtures/deploy_with_group_strategy.yml index bc551f8a..a082f539 100644 --- a/test/fixtures/deploy_with_group_strategy.yml +++ b/test/fixtures/deploy_with_group_strategy.yml @@ -12,5 +12,6 @@ registry: username: user password: pw -group_limit: 3 -group_wait: 30 +boot: + group_limit: 3 + group_wait: 30 From f055766918f9fe9fa4b91192c937f26dc61bcd5a Mon Sep 17 00:00:00 2001 From: Kevin McConnell Date: Fri, 14 Apr 2023 11:26:10 +0100 Subject: [PATCH 4/5] Allow percentage-based rolling deployments --- README.md | 2 +- lib/mrsk/configuration.rb | 2 +- lib/mrsk/configuration/boot.rb | 20 ++++++++++---- test/cli/app_test.rb | 2 +- test/commander_test.rb | 27 ++++++++++++++++--- test/fixtures/deploy_with_group_strategy.yml | 2 +- .../deploy_with_precentage_group_strategy.yml | 17 ++++++++++++ 7 files changed, 60 insertions(+), 12 deletions(-) create mode 100644 test/fixtures/deploy_with_precentage_group_strategy.yml diff --git a/README.md b/README.md index 9fcea1ed..093d43bf 100644 --- a/README.md +++ b/README.md @@ -841,7 +841,7 @@ MRSK's default is to boot new containers on all hosts in parallel. But you can c service: myservice boot: - group_limit: 10 + group_limit: 10 # Can also specify as a percentage of total hosts, such as "25%" group_wait: 2 ``` diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index c3415703..4d2d010a 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -88,7 +88,7 @@ class Mrsk::Configuration end def boot - Mrsk::Configuration::Boot.new(section: raw_config.boot) + Mrsk::Configuration::Boot.new(config: self) end diff --git a/lib/mrsk/configuration/boot.rb b/lib/mrsk/configuration/boot.rb index 150f3d5f..dd86b689 100644 --- a/lib/mrsk/configuration/boot.rb +++ b/lib/mrsk/configuration/boot.rb @@ -1,9 +1,19 @@ class Mrsk::Configuration::Boot - attr_reader :group_wait, :group_limit + def initialize(config:) + @options = config.raw_config.boot || {} + @host_count = config.all_hosts.count + end - def initialize(section:) - section = section || {} - @group_limit = section["group_limit"] - @group_wait = section["group_wait"] + def group_limit + limit = @options["group_limit"] + if limit.to_s.end_with?("%") + @host_count * limit.to_i / 100 + else + limit + end + end + + def group_wait + @options["group_wait"] end end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index f27c04ea..dc2594b9 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -38,7 +38,7 @@ class CliAppTest < CliTestCase Mrsk::Cli::App.any_instance.stubs(:on).with([ "1.1.1.1" ]) # tag container # Strategy is used when booting the containers - Mrsk::Cli::App.any_instance.expects(:on).with([ "1.1.1.1" ], in: :groups, limit: 3, wait: 30).with_block_given + Mrsk::Cli::App.any_instance.expects(:on).with([ "1.1.1.1" ], in: :groups, limit: 3, wait: 2).with_block_given run_command("boot", config: :with_group_strategy) end diff --git a/test/commander_test.rb b/test/commander_test.rb index 163feaf7..e5fd2961 100644 --- a/test/commander_test.rb +++ b/test/commander_test.rb @@ -2,9 +2,7 @@ require "test_helper" class CommanderTest < ActiveSupport::TestCase setup do - @mrsk = Mrsk::Commander.new.tap do |mrsk| - mrsk.configure config_file: Pathname.new(File.expand_path("fixtures/deploy_with_roles.yml", __dir__)) - end + configure_with(:deploy_with_roles) end test "lazy configuration" do @@ -55,4 +53,27 @@ class CommanderTest < ActiveSupport::TestCase assert_equal [ "web" ], @mrsk.roles_on("1.1.1.1") assert_equal [ "workers" ], @mrsk.roles_on("1.1.1.3") end + + test "default group strategy" do + assert_empty @mrsk.group_strategy + end + + test "specific limit group strategy" do + configure_with(:deploy_with_group_strategy) + + assert_equal({ in: :groups, limit: 3, wait: 2 }, @mrsk.group_strategy) + end + + test "percentage-based group strategy" do + configure_with(:deploy_with_precentage_group_strategy) + + assert_equal({ in: :groups, limit: 1, wait: 2 }, @mrsk.group_strategy) + end + + private + def configure_with(variant) + @mrsk = Mrsk::Commander.new.tap do |mrsk| + mrsk.configure config_file: Pathname.new(File.expand_path("fixtures/#{variant}.yml", __dir__)) + end + end end diff --git a/test/fixtures/deploy_with_group_strategy.yml b/test/fixtures/deploy_with_group_strategy.yml index a082f539..91ae3cc0 100644 --- a/test/fixtures/deploy_with_group_strategy.yml +++ b/test/fixtures/deploy_with_group_strategy.yml @@ -14,4 +14,4 @@ registry: boot: group_limit: 3 - group_wait: 30 + group_wait: 2 diff --git a/test/fixtures/deploy_with_precentage_group_strategy.yml b/test/fixtures/deploy_with_precentage_group_strategy.yml new file mode 100644 index 00000000..e738d07d --- /dev/null +++ b/test/fixtures/deploy_with_precentage_group_strategy.yml @@ -0,0 +1,17 @@ +service: app +image: dhh/app +servers: + web: + - "1.1.1.1" + - "1.1.1.2" + workers: + - "1.1.1.3" + - "1.1.1.4" + +registry: + username: user + password: pw + +boot: + group_limit: 25% + group_wait: 2 From c83b74dcb74d4388555b0a8f6c426e68216c223a Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Tue, 2 May 2023 13:11:31 +0200 Subject: [PATCH 5/5] Simplify domain language to just "boot" and unscoped config keys --- README.md | 8 ++++---- lib/mrsk/cli/app.rb | 2 +- lib/mrsk/commander.rb | 6 +++--- lib/mrsk/configuration/boot.rb | 9 +++++---- test/cli/app_test.rb | 2 +- test/commander_test.rb | 10 +++++----- ...roup_strategy.yml => deploy_with_boot_strategy.yml} | 4 ++-- ...gy.yml => deploy_with_precentage_boot_strategy.yml} | 4 ++-- 8 files changed, 23 insertions(+), 22 deletions(-) rename test/fixtures/{deploy_with_group_strategy.yml => deploy_with_boot_strategy.yml} (83%) rename test/fixtures/{deploy_with_precentage_group_strategy.yml => deploy_with_precentage_boot_strategy.yml} (82%) diff --git a/README.md b/README.md index 093d43bf..4cb298fa 100644 --- a/README.md +++ b/README.md @@ -835,17 +835,17 @@ mrsk lock release When deploying to large numbers of hosts, you might prefer not to restart your services on every host at the same time. -MRSK's default is to boot new containers on all hosts in parallel. But you can control this by configuring `group_limit` and `group_wait` as boot options: +MRSK's default is to boot new containers on all hosts in parallel. But you can control this by configuring `boot/limit` and `boot/wait` as options: ```yaml service: myservice boot: - group_limit: 10 # Can also specify as a percentage of total hosts, such as "25%" - group_wait: 2 + limit: 10 # Can also specify as a percentage of total hosts, such as "25%" + wait: 2 ``` -When `group_limit` is specified, containers will be booted on, at most, `group_limit` hosts at once. MRSK will pause for `group_wait` seconds between batches. +When `limit` is specified, containers will be booted on, at most, `limit` hosts at once. MRSK will pause for `wait` seconds between batches. These settings only apply when booting containers (using `mrsk deploy`, or `mrsk app boot`). For other commands, MRSK continues to run commands in parallel across all hosts. diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb index 7f6fbf44..3722ce6e 100644 --- a/lib/mrsk/cli/app.rb +++ b/lib/mrsk/cli/app.rb @@ -13,7 +13,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base execute *MRSK.app.tag_current_as_latest end - on(MRSK.hosts, **MRSK.group_strategy) do |host| + on(MRSK.hosts, **MRSK.boot_strategy) do |host| roles = MRSK.roles_on(host) roles.each do |role| diff --git a/lib/mrsk/commander.rb b/lib/mrsk/commander.rb index 160b619e..217918e6 100644 --- a/lib/mrsk/commander.rb +++ b/lib/mrsk/commander.rb @@ -51,9 +51,9 @@ class Mrsk::Commander end end - def group_strategy - if config.boot.group_limit.present? - { in: :groups, limit: config.boot.group_limit, wait: config.boot.group_wait } + def boot_strategy + if config.boot.limit.present? + { in: :groups, limit: config.boot.limit, wait: config.boot.wait } else {} end diff --git a/lib/mrsk/configuration/boot.rb b/lib/mrsk/configuration/boot.rb index dd86b689..1332398a 100644 --- a/lib/mrsk/configuration/boot.rb +++ b/lib/mrsk/configuration/boot.rb @@ -4,8 +4,9 @@ class Mrsk::Configuration::Boot @host_count = config.all_hosts.count end - def group_limit - limit = @options["group_limit"] + def limit + limit = @options["limit"] + if limit.to_s.end_with?("%") @host_count * limit.to_i / 100 else @@ -13,7 +14,7 @@ class Mrsk::Configuration::Boot end end - def group_wait - @options["group_wait"] + def wait + @options["wait"] end end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index dc2594b9..3dddb078 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -40,7 +40,7 @@ class CliAppTest < CliTestCase # Strategy is used when booting the containers Mrsk::Cli::App.any_instance.expects(:on).with([ "1.1.1.1" ], in: :groups, limit: 3, wait: 2).with_block_given - run_command("boot", config: :with_group_strategy) + run_command("boot", config: :with_boot_strategy) end test "start" do diff --git a/test/commander_test.rb b/test/commander_test.rb index e5fd2961..25dfabdd 100644 --- a/test/commander_test.rb +++ b/test/commander_test.rb @@ -55,19 +55,19 @@ class CommanderTest < ActiveSupport::TestCase end test "default group strategy" do - assert_empty @mrsk.group_strategy + assert_empty @mrsk.boot_strategy end test "specific limit group strategy" do - configure_with(:deploy_with_group_strategy) + configure_with(:deploy_with_boot_strategy) - assert_equal({ in: :groups, limit: 3, wait: 2 }, @mrsk.group_strategy) + assert_equal({ in: :groups, limit: 3, wait: 2 }, @mrsk.boot_strategy) end test "percentage-based group strategy" do - configure_with(:deploy_with_precentage_group_strategy) + configure_with(:deploy_with_precentage_boot_strategy) - assert_equal({ in: :groups, limit: 1, wait: 2 }, @mrsk.group_strategy) + assert_equal({ in: :groups, limit: 1, wait: 2 }, @mrsk.boot_strategy) end private diff --git a/test/fixtures/deploy_with_group_strategy.yml b/test/fixtures/deploy_with_boot_strategy.yml similarity index 83% rename from test/fixtures/deploy_with_group_strategy.yml rename to test/fixtures/deploy_with_boot_strategy.yml index 91ae3cc0..7691eb2e 100644 --- a/test/fixtures/deploy_with_group_strategy.yml +++ b/test/fixtures/deploy_with_boot_strategy.yml @@ -13,5 +13,5 @@ registry: password: pw boot: - group_limit: 3 - group_wait: 2 + limit: 3 + wait: 2 diff --git a/test/fixtures/deploy_with_precentage_group_strategy.yml b/test/fixtures/deploy_with_precentage_boot_strategy.yml similarity index 82% rename from test/fixtures/deploy_with_precentage_group_strategy.yml rename to test/fixtures/deploy_with_precentage_boot_strategy.yml index e738d07d..eb68a52f 100644 --- a/test/fixtures/deploy_with_precentage_group_strategy.yml +++ b/test/fixtures/deploy_with_precentage_boot_strategy.yml @@ -13,5 +13,5 @@ registry: password: pw boot: - group_limit: 25% - group_wait: 2 + limit: 25% + wait: 2