diff --git a/README.md b/README.md index 95042cd9..8ae2121c 100644 --- a/README.md +++ b/README.md @@ -855,6 +855,24 @@ mrsk lock acquire -m "Doing maintanence" mrsk lock release ``` +## Rolling deployments + +When deploying to large numbers of hosts, you might prefer not to restart your services on every host at the same time. + +MRSK's default is to boot new containers on all hosts in parallel. But you can control this by configuring `boot/limit` and `boot/wait` as options: + +```yaml +service: myservice + +boot: + limit: 10 # Can also specify as a percentage of total hosts, such as "25%" + wait: 2 +``` + +When `limit` is specified, containers will be booted on, at most, `limit` hosts at once. MRSK will pause for `wait` seconds between batches. + +These settings only apply when booting containers (using `mrsk deploy`, or `mrsk app boot`). For other commands, MRSK continues to run commands in parallel across all hosts. + ## Stage of development This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com). diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb index 6e4fd496..9e783efb 100644 --- a/lib/mrsk/cli/app.rb +++ b/lib/mrsk/cli/app.rb @@ -11,7 +11,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base execute *MRSK.app.tag_current_as_latest end - on(MRSK.hosts) do |host| + on(MRSK.hosts, **MRSK.boot_strategy) do |host| roles = MRSK.roles_on(host) roles.each do |role| diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index 96c78098..a7832e73 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -233,15 +233,24 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base subcommand "lock", Mrsk::Cli::Lock private - def container_available?(version, host: MRSK.primary_host) - available = nil - - on(host) do - first_role = MRSK.roles_on(host).first - available = capture_with_info(*MRSK.app(role: first_role).container_id_for_version(version)).present? + def container_available?(version) + begin + on(MRSK.hosts) do + MRSK.roles_on(host).each do |role| + container_id = capture_with_info(*MRSK.app(role: role).container_id_for_version(version)) + raise "Container not found" unless container_id.present? + end + end + rescue SSHKit::Runner::ExecuteError => e + if e.message =~ /Container not found/ + say "Error looking for container version #{version}: #{e.message}" + return false + else + raise + end end - available + true end def deploy_options diff --git a/lib/mrsk/cli/prune.rb b/lib/mrsk/cli/prune.rb index 6698ba27..bcfdd5bf 100644 --- a/lib/mrsk/cli/prune.rb +++ b/lib/mrsk/cli/prune.rb @@ -7,7 +7,7 @@ class Mrsk::Cli::Prune < Mrsk::Cli::Base end end - desc "images", "Prune unused images older than 7 days" + desc "images", "Prune dangling images" def images with_lock do on(MRSK.hosts) do @@ -17,7 +17,7 @@ class Mrsk::Cli::Prune < Mrsk::Cli::Base end end - desc "containers", "Prune stopped containers older than 3 days" + desc "containers", "Prune all stopped containers, except the last 5" def containers with_lock do on(MRSK.hosts) do diff --git a/lib/mrsk/commander.rb b/lib/mrsk/commander.rb index 14b2c2f8..99a0f7bb 100644 --- a/lib/mrsk/commander.rb +++ b/lib/mrsk/commander.rb @@ -51,6 +51,14 @@ class Mrsk::Commander end end + def boot_strategy + if config.boot.limit.present? + { in: :groups, limit: config.boot.limit, wait: config.boot.wait } + else + {} + end + end + def roles_on(host) roles.select { |role| role.hosts.include?(host.to_s) }.map(&:name) end diff --git a/lib/mrsk/commands/prune.rb b/lib/mrsk/commands/prune.rb index 71218cda..0ac779d5 100644 --- a/lib/mrsk/commands/prune.rb +++ b/lib/mrsk/commands/prune.rb @@ -2,11 +2,19 @@ require "active_support/duration" require "active_support/core_ext/numeric/time" class Mrsk::Commands::Prune < Mrsk::Commands::Base - def images(until_hours: 7.days.in_hours.to_i) - docker :image, :prune, "--all", "--force", "--filter", "label=service=#{config.service}", "--filter", "until=#{until_hours}h" + def images + docker :image, :prune, "--all", "--force", "--filter", "label=service=#{config.service}", "--filter", "dangling=true" end - def containers(until_hours: 3.days.in_hours.to_i) - docker :container, :prune, "--force", "--filter", "label=service=#{config.service}", "--filter", "until=#{until_hours}h" + def containers(keep_last: 5) + pipe \ + docker(:ps, "-q", "-a", "--filter", "label=service=#{config.service}", *stopped_containers_filters), + "tail -n +#{keep_last + 1}", + "while read container_id; do docker rm $container_id; done" end + + private + def stopped_containers_filters + [ "created", "exited", "dead" ].flat_map { |status| ["--filter", "status=#{status}"] } + end end diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index a2c2ecf1..4d2d010a 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -87,6 +87,10 @@ class Mrsk::Configuration roles.select(&:running_traefik?).flat_map(&:hosts).uniq end + def boot + Mrsk::Configuration::Boot.new(config: self) + end + def repository [ raw_config.registry["server"], image ].compact.join("/") diff --git a/lib/mrsk/configuration/boot.rb b/lib/mrsk/configuration/boot.rb new file mode 100644 index 00000000..1332398a --- /dev/null +++ b/lib/mrsk/configuration/boot.rb @@ -0,0 +1,20 @@ +class Mrsk::Configuration::Boot + def initialize(config:) + @options = config.raw_config.boot || {} + @host_count = config.all_hosts.count + end + + def limit + limit = @options["limit"] + + if limit.to_s.end_with?("%") + @host_count * limit.to_i / 100 + else + limit + end + end + + def wait + @options["wait"] + end +end diff --git a/lib/mrsk/configuration/role.rb b/lib/mrsk/configuration/role.rb index 881b331e..55b69731 100644 --- a/lib/mrsk/configuration/role.rb +++ b/lib/mrsk/configuration/role.rb @@ -89,6 +89,9 @@ class Mrsk::Configuration::Role def traefik_labels if running_traefik? { + # Setting a service property ensures that the generated service name will be consistent between versions + "traefik.http.services.#{traefik_service}.loadbalancer.server.scheme" => "http", + "traefik.http.routers.#{traefik_service}.rule" => "PathPrefix(`/`)", "traefik.http.middlewares.#{traefik_service}-retry.retry.attempts" => "5", "traefik.http.middlewares.#{traefik_service}-retry.retry.initialinterval" => "500ms", diff --git a/lib/mrsk/utils/healthcheck_poller.rb b/lib/mrsk/utils/healthcheck_poller.rb index d7b8be65..3ef5b7a8 100644 --- a/lib/mrsk/utils/healthcheck_poller.rb +++ b/lib/mrsk/utils/healthcheck_poller.rb @@ -1,5 +1,5 @@ class Mrsk::Utils::HealthcheckPoller - TRAEFIK_HEALTHY_DELAY = 1 + TRAEFIK_HEALTHY_DELAY = 2 class HealthcheckError < StandardError; end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 54292d65..977f2900 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -40,6 +40,16 @@ class CliAppTest < CliTestCase Thread.report_on_exception = true end + test "boot uses group strategy when specified" do + Mrsk::Cli::App.any_instance.stubs(:on).with("1.1.1.1").twice # acquire & release lock + Mrsk::Cli::App.any_instance.stubs(:on).with([ "1.1.1.1" ]) # tag container + + # Strategy is used when booting the containers + Mrsk::Cli::App.any_instance.expects(:on).with([ "1.1.1.1" ], in: :groups, limit: 3, wait: 2).with_block_given + + run_command("boot", config: :with_boot_strategy) + end + test "start" do run_command("start").tap do |output| assert_match "docker start app-web-999", output @@ -158,7 +168,7 @@ class CliAppTest < CliTestCase end private - def run_command(*command) - stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1"]) } + def run_command(*command, config: :with_accessories) + stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1"]) } end end diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index bcd72813..746b9c9a 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -140,7 +140,8 @@ class CliMainTest < CliTestCase end test "rollback bad version" do - # Mrsk::Cli::Main.any_instance.stubs(:container_available?).returns(false) + Thread.report_on_exception = false + run_command("details") # Preheat MRSK const run_command("rollback", "nonsense").tap do |output| @@ -150,9 +151,19 @@ class CliMainTest < CliTestCase end test "rollback good version" do - Mrsk::Cli::Main.any_instance.stubs(:container_available?).returns(true) - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-").returns("version-to-rollback\n").at_least_once - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=workers", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-").returns("version-to-rollback\n").at_least_once + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet") + .returns("version-to-rollback\n").at_least_once + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-123$", "--quiet") + .returns("version-to-rollback\n").at_least_once + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-") + .returns("version-to-rollback\n").at_least_once + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=workers", "--filter", "status=running", "--latest", "--format", "\"{{.Names}}\"", "|", "grep -oE \"\\-[^-]+$\"", "|", "cut -c 2-") + .returns("version-to-rollback\n").at_least_once + run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output| assert_match "Start version 123", output diff --git a/test/cli/prune_test.rb b/test/cli/prune_test.rb index 4ff9eedc..40a1d80f 100644 --- a/test/cli/prune_test.rb +++ b/test/cli/prune_test.rb @@ -10,13 +10,13 @@ class CliPruneTest < CliTestCase test "images" do run_command("images").tap do |output| - assert_match /docker image prune --all --force --filter label=service=app --filter until=168h on 1.1.1.\d/, output + assert_match /docker image prune --all --force --filter label=service=app --filter dangling=true on 1.1.1.\d/, output end end test "containers" do run_command("containers").tap do |output| - assert_match /docker container prune --force --filter label=service=app --filter until=72h on 1.1.1.\d/, output + assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output end end diff --git a/test/commander_test.rb b/test/commander_test.rb index 163feaf7..25dfabdd 100644 --- a/test/commander_test.rb +++ b/test/commander_test.rb @@ -2,9 +2,7 @@ require "test_helper" class CommanderTest < ActiveSupport::TestCase setup do - @mrsk = Mrsk::Commander.new.tap do |mrsk| - mrsk.configure config_file: Pathname.new(File.expand_path("fixtures/deploy_with_roles.yml", __dir__)) - end + configure_with(:deploy_with_roles) end test "lazy configuration" do @@ -55,4 +53,27 @@ class CommanderTest < ActiveSupport::TestCase assert_equal [ "web" ], @mrsk.roles_on("1.1.1.1") assert_equal [ "workers" ], @mrsk.roles_on("1.1.1.3") end + + test "default group strategy" do + assert_empty @mrsk.boot_strategy + end + + test "specific limit group strategy" do + configure_with(:deploy_with_boot_strategy) + + assert_equal({ in: :groups, limit: 3, wait: 2 }, @mrsk.boot_strategy) + end + + test "percentage-based group strategy" do + configure_with(:deploy_with_precentage_boot_strategy) + + assert_equal({ in: :groups, limit: 1, wait: 2 }, @mrsk.boot_strategy) + end + + private + def configure_with(variant) + @mrsk = Mrsk::Commander.new.tap do |mrsk| + mrsk.configure config_file: Pathname.new(File.expand_path("fixtures/#{variant}.yml", __dir__)) + end + end end diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index f74b21c9..44404558 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -13,7 +13,7 @@ class CommandsAppTest < ActiveSupport::TestCase test "run" do assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -21,7 +21,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:volumes] = ["/local/path:/container/path" ] assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -29,7 +29,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:healthcheck] = { "path" => "/healthz" } assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/healthz || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/healthz || exit 1\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -37,7 +37,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:healthcheck] = { "cmd" => "/bin/up" } assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"/bin/up\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"/bin/up\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -45,7 +45,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } } assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"/bin/healthy\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"/bin/healthy\" --health-interval \"1s\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -60,7 +60,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end diff --git a/test/commands/prune_test.rb b/test/commands/prune_test.rb index adbe2816..bd6a561d 100644 --- a/test/commands/prune_test.rb +++ b/test/commands/prune_test.rb @@ -10,13 +10,13 @@ class CommandsPruneTest < ActiveSupport::TestCase test "images" do assert_equal \ - "docker image prune --all --force --filter label=service=app --filter until=168h", + "docker image prune --all --force --filter label=service=app --filter dangling=true", new_command.images.join(" ") end test "containers" do assert_equal \ - "docker container prune --force --filter label=service=app --filter until=72h", + "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done", new_command.containers.join(" ") end diff --git a/test/configuration/role_test.rb b/test/configuration/role_test.rb index f3814066..7708bd9d 100644 --- a/test/configuration/role_test.rb +++ b/test/configuration/role_test.rb @@ -42,7 +42,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase end test "special label args for web" do - assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args + assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args end test "custom labels" do @@ -66,7 +66,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase c[:servers]["beta"] = { "traefik" => "true", "hosts" => [ "1.1.1.5" ] } }) - assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args + assert_equal [ "--label", "service=\"app\"", "--label", "role=\"beta\"", "--label", "traefik.http.services.app-beta.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-beta.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-beta-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-beta.middlewares=\"app-beta-retry@docker\"" ], config.role(:beta).label_args end test "env overwritten by role" do diff --git a/test/fixtures/deploy_with_boot_strategy.yml b/test/fixtures/deploy_with_boot_strategy.yml new file mode 100644 index 00000000..7691eb2e --- /dev/null +++ b/test/fixtures/deploy_with_boot_strategy.yml @@ -0,0 +1,17 @@ +service: app +image: dhh/app +servers: + web: + - "1.1.1.1" + - "1.1.1.2" + workers: + - "1.1.1.3" + - "1.1.1.4" + +registry: + username: user + password: pw + +boot: + limit: 3 + wait: 2 diff --git a/test/fixtures/deploy_with_precentage_boot_strategy.yml b/test/fixtures/deploy_with_precentage_boot_strategy.yml new file mode 100644 index 00000000..eb68a52f --- /dev/null +++ b/test/fixtures/deploy_with_precentage_boot_strategy.yml @@ -0,0 +1,17 @@ +service: app +image: dhh/app +servers: + web: + - "1.1.1.1" + - "1.1.1.2" + workers: + - "1.1.1.3" + - "1.1.1.4" + +registry: + username: user + password: pw + +boot: + limit: 25% + wait: 2 diff --git a/test/integration/deploy_test.rb b/test/integration/deploy_test.rb index 89b2269f..351dc5d2 100644 --- a/test/integration/deploy_test.rb +++ b/test/integration/deploy_test.rb @@ -51,7 +51,15 @@ class DeployTest < ActiveSupport::TestCase end def assert_app_is_up - assert_equal "200", app_response.code + code = app_response.code + if code != "200" + puts "Got response code #{code}, here are the traefik logs:" + mrsk :traefik, :logs + puts "And here are the load balancer logs" + docker_compose :logs, :load_balancer + puts "Tried to get the response code again and got #{app_response.code}" + end + assert_equal "200", code end def app_response @@ -61,7 +69,10 @@ class DeployTest < ActiveSupport::TestCase def wait_for_healthy(timeout: 20) timeout_at = Time.now + timeout while docker_compose("ps -a | tail -n +2 | grep -v '(healthy)' | wc -l", capture: true) != "0" - raise "Container not healthy after #{timeout} seconds" if timeout_at < Time.now + if timeout_at < Time.now + docker_compose("ps -a | tail -n +2 | grep -v '(healthy)'") + raise "Container not healthy after #{timeout} seconds" if timeout_at < Time.now + end sleep 0.1 end end diff --git a/test/integration/docker/deployer/Dockerfile b/test/integration/docker/deployer/Dockerfile index 22556cc2..ccd1de39 100644 --- a/test/integration/docker/deployer/Dockerfile +++ b/test/integration/docker/deployer/Dockerfile @@ -2,7 +2,7 @@ FROM ruby:3.2 WORKDIR /app -RUN apt-get update && apt-get install -y ca-certificates openssh-client curl gnupg docker.io +RUN apt-get update --fix-missing && apt-get install -y ca-certificates openssh-client curl gnupg docker.io RUN install -m 0755 -d /etc/apt/keyrings RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg @@ -12,7 +12,7 @@ RUN echo \ "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ tee /etc/apt/sources.list.d/docker.list > /dev/null -RUN apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +RUN apt-get update --fix-missing && apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin COPY boot.sh . COPY app/ . diff --git a/test/integration/docker/deployer/app/config/deploy.yml b/test/integration/docker/deployer/app/config/deploy.yml index 5ac25b14..8442c57b 100644 --- a/test/integration/docker/deployer/app/config/deploy.yml +++ b/test/integration/docker/deployer/app/config/deploy.yml @@ -10,5 +10,8 @@ registry: builder: multiarch: false healthcheck: - path: / - port: 80 + cmd: wget -qO- http://localhost > /dev/null +traefik: + args: + accesslog: true + accesslog.format: json diff --git a/test/integration/docker/shared/Dockerfile b/test/integration/docker/shared/Dockerfile index dae69053..bc0d8e84 100644 --- a/test/integration/docker/shared/Dockerfile +++ b/test/integration/docker/shared/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:22.10 WORKDIR /work -RUN apt-get update && apt-get -y install openssh-client openssl +RUN apt-get update --fix-missing && apt-get -y install openssh-client openssl RUN mkdir ssh && \ ssh-keygen -t rsa -f ssh/id_rsa -N "" diff --git a/test/integration/docker/vm/Dockerfile b/test/integration/docker/vm/Dockerfile index 99f881fa..f481023c 100644 --- a/test/integration/docker/vm/Dockerfile +++ b/test/integration/docker/vm/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:22.10 WORKDIR /work -RUN apt-get update && apt-get -y install openssh-client openssh-server docker.io +RUN apt-get update --fix-missing && apt-get -y install openssh-client openssh-server docker.io RUN mkdir /root/.ssh && ln -s /shared/ssh/id_rsa.pub /root/.ssh/authorized_keys RUN mkdir -p /etc/docker/certs.d/registry:4443 && ln -s /shared/certs/domain.crt /etc/docker/certs.d/registry:4443/ca.crt