From e178907a214143f2325ce46f0dd89880f0e3c1b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20B=C3=BChlmann?= Date: Thu, 9 Mar 2023 16:46:57 +0100 Subject: [PATCH 01/32] Don't list duplicate hosts --- lib/mrsk/configuration.rb | 4 ++-- test/configuration_test.rb | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index 64c116a2..e1e15f94 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -64,7 +64,7 @@ class Mrsk::Configuration def all_hosts - roles.flat_map(&:hosts) + roles.flat_map(&:hosts).uniq end def primary_web_host @@ -72,7 +72,7 @@ class Mrsk::Configuration end def traefik_hosts - roles.select(&:running_traefik?).flat_map(&:hosts) + roles.select(&:running_traefik?).flat_map(&:hosts).uniq end diff --git a/test/configuration_test.rb b/test/configuration_test.rb index 5cd7856b..9c1aefec 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -15,7 +15,7 @@ class ConfigurationTest < ActiveSupport::TestCase @config = Mrsk::Configuration.new(@deploy) @deploy_with_roles = @deploy.dup.merge({ - servers: { "web" => [ "1.1.1.1", "1.1.1.2" ], "workers" => { "hosts" => [ "1.1.1.3", "1.1.1.4" ] } } }) + servers: { "web" => [ "1.1.1.1", "1.1.1.2" ], "workers" => { "hosts" => [ "1.1.1.1", "1.1.1.3" ] } } }) @config_with_roles = Mrsk::Configuration.new(@deploy_with_roles) end @@ -48,7 +48,7 @@ class ConfigurationTest < ActiveSupport::TestCase test "all hosts" do assert_equal [ "1.1.1.1", "1.1.1.2"], @config.all_hosts - assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @config_with_roles.all_hosts + assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], @config_with_roles.all_hosts end test "primary web host" do @@ -62,7 +62,7 @@ class ConfigurationTest < ActiveSupport::TestCase @deploy_with_roles[:servers]["workers"]["traefik"] = true config = Mrsk::Configuration.new(@deploy_with_roles) - assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], config.traefik_hosts + assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], config.traefik_hosts end test "version" do From 901484d75d5007b59349d4978ce45f8dae1bbd07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20B=C3=BChlmann?= Date: Thu, 9 Mar 2023 18:21:39 +0100 Subject: [PATCH 02/32] Filter roles and hosts by their respective counterpart --- lib/mrsk/commander.rb | 22 ++++++++++++++--- test/commander_test.rb | 37 +++++++++++++++++++++-------- test/fixtures/deploy_with_roles.yml | 2 +- 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/lib/mrsk/commander.rb b/lib/mrsk/commander.rb index be2fd2e3..ad61e895 100644 --- a/lib/mrsk/commander.rb +++ b/lib/mrsk/commander.rb @@ -14,22 +14,38 @@ class Mrsk::Commander .tap { |config| configure_sshkit_with(config) } end - attr_accessor :specific_hosts + attr_reader :specific_roles, :specific_hosts def specific_primary! self.specific_hosts = [ config.primary_web_host ] end def specific_roles=(role_names) - self.specific_hosts = config.roles.select { |r| role_names.include?(r.name) }.flat_map(&:hosts) if role_names.present? + @specific_roles = config.roles.select { |r| role_names.include?(r.name) } if role_names.present? + end + + def specific_hosts=(hosts) + @specific_hosts = config.all_hosts & hosts if hosts.present? end def primary_host specific_hosts&.first || config.primary_web_host end + def roles + (specific_roles || config.roles).select do |role| + ((specific_hosts || config.all_hosts) & role.hosts).any? + end + end + def hosts - specific_hosts || config.all_hosts + (specific_hosts || config.all_hosts).select do |host| + (specific_roles || config.roles).flat_map(&:hosts).include?(host) + end + end + + def roles_on(host) + roles.select { |role| role.hosts.include?(host) } end def traefik_hosts diff --git a/test/commander_test.rb b/test/commander_test.rb index b4ce24dc..918f69f8 100644 --- a/test/commander_test.rb +++ b/test/commander_test.rb @@ -19,25 +19,36 @@ class CommanderTest < ActiveSupport::TestCase assert_match /no git repository found/, error.message end - test "overwriting hosts" do - assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @mrsk.hosts + test "filtering hosts" do + assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], @mrsk.hosts - @mrsk.specific_hosts = [ "1.2.3.4", "1.2.3.5" ] - assert_equal [ "1.2.3.4", "1.2.3.5" ], @mrsk.hosts + @mrsk.specific_hosts = [ "1.1.1.1", "1.1.1.2" ] + assert_equal [ "1.1.1.1", "1.1.1.2" ], @mrsk.hosts end - test "overwriting hosts with roles" do - assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @mrsk.hosts + test "filtering hosts by filtering roles" do + assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], @mrsk.hosts - @mrsk.specific_roles = [ "workers", "web" ] - assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @mrsk.hosts + @mrsk.specific_roles = [ "web" ] + assert_equal [ "1.1.1.1", "1.1.1.2" ], @mrsk.hosts + end + + test "filtering roles" do + assert_equal [ "web", "workers" ], @mrsk.roles.map(&:name) @mrsk.specific_roles = [ "workers" ] - assert_equal [ "1.1.1.3", "1.1.1.4" ], @mrsk.hosts + assert_equal [ "workers" ], @mrsk.roles.map(&:name) + end + + test "filtering roles by filtering hosts" do + assert_equal [ "web", "workers" ], @mrsk.roles.map(&:name) + + @mrsk.specific_hosts = [ "1.1.1.3" ] + assert_equal [ "workers" ], @mrsk.roles.map(&:name) end test "overwriting hosts with primary" do - assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @mrsk.hosts + assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], @mrsk.hosts @mrsk.specific_primary! assert_equal [ "1.1.1.1" ], @mrsk.hosts @@ -47,4 +58,10 @@ class CommanderTest < ActiveSupport::TestCase @mrsk.specific_roles = "web" assert_equal "1.1.1.1", @mrsk.primary_host end + + test "roles_on" do + assert_equal [ "web", "workers" ], @mrsk.roles_on("1.1.1.1").map(&:name) + assert_equal [ "web" ], @mrsk.roles_on("1.1.1.2").map(&:name) + assert_equal [ "workers" ], @mrsk.roles_on("1.1.1.3").map(&:name) + end end diff --git a/test/fixtures/deploy_with_roles.yml b/test/fixtures/deploy_with_roles.yml index 345c6301..bac32e44 100644 --- a/test/fixtures/deploy_with_roles.yml +++ b/test/fixtures/deploy_with_roles.yml @@ -5,8 +5,8 @@ servers: - 1.1.1.1 - 1.1.1.2 workers: + - 1.1.1.1 - 1.1.1.3 - - 1.1.1.4 env: REDIS_URL: redis://x/y registry: From 6b1130323048dc867b0a391f7a356baf3dbf1f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20B=C3=BChlmann?= Date: Thu, 9 Mar 2023 19:35:30 +0100 Subject: [PATCH 03/32] Prepare auditor to print a present role --- lib/mrsk/commander.rb | 4 ++-- lib/mrsk/commands/auditor.rb | 19 +++++++++++++++++-- test/commands/auditor_test.rb | 10 +++++++++- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/lib/mrsk/commander.rb b/lib/mrsk/commander.rb index ad61e895..d49243c3 100644 --- a/lib/mrsk/commander.rb +++ b/lib/mrsk/commander.rb @@ -69,8 +69,8 @@ class Mrsk::Commander Mrsk::Commands::Accessory.new(config, name: name) end - def auditor - @auditor ||= Mrsk::Commands::Auditor.new(config) + def auditor(role: nil) + Mrsk::Commands::Auditor.new(config, role: role) end def builder diff --git a/lib/mrsk/commands/auditor.rb b/lib/mrsk/commands/auditor.rb index a0bc0076..9d6cf414 100644 --- a/lib/mrsk/commands/auditor.rb +++ b/lib/mrsk/commands/auditor.rb @@ -1,6 +1,13 @@ require "active_support/core_ext/time/conversions" class Mrsk::Commands::Auditor < Mrsk::Commands::Base + attr_reader :role + + def initialize(config, role: nil) + super(config) + @role = role + end + # Runs remotely def record(line) append \ @@ -25,11 +32,15 @@ class Mrsk::Commands::Auditor < Mrsk::Commands::Base end def tagged_record_line(line) - "'#{recorded_at_tag} #{performer_tag} #{line}'" + quote [recorded_at_tag, performer_tag, role_tag, line].compact.join(" ") end def tagged_broadcast_line(line) - "'#{performer_tag} #{line}'" + quote [performer_tag, role_tag, line].compact.join(" ") + end + + def role_tag + "[#{role}]" if role end def performer_tag @@ -39,4 +50,8 @@ class Mrsk::Commands::Auditor < Mrsk::Commands::Base def recorded_at_tag "[#{Time.now.to_fs(:db)}]" end + + def quote(tagged_line) + "'#{tagged_line}'" + end end diff --git a/test/commands/auditor_test.rb b/test/commands/auditor_test.rb index f35e110c..55c3a32e 100644 --- a/test/commands/auditor_test.rb +++ b/test/commands/auditor_test.rb @@ -22,6 +22,14 @@ class CommandsAuditorTest < ActiveSupport::TestCase new_command.record("app removed container").join(" ") end + test "record with role" do + @role = "web" + + assert_match \ + /echo '.* \[web\] app removed container' >> mrsk-app-audit.log/, + new_command.record("app removed container").join(" ") + end + test "broadcast" do assert_match \ /bin\/audit_broadcast '\[.*\] app removed container'/, @@ -30,6 +38,6 @@ class CommandsAuditorTest < ActiveSupport::TestCase private def new_command - Mrsk::Commands::Auditor.new(Mrsk::Configuration.new(@config, destination: @destination, version: "123")) + Mrsk::Commands::Auditor.new(Mrsk::Configuration.new(@config, destination: @destination, version: "123"), role: @role) end end From fdb0c8ee91360b46975685c809a4d7b95054ef42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20B=C3=BChlmann?= Date: Fri, 10 Mar 2023 08:50:26 +0100 Subject: [PATCH 04/32] Rolify app cli/command --- lib/mrsk/cli/app.rb | 80 +++++++++++++++-------- lib/mrsk/commander.rb | 6 +- lib/mrsk/commands/app.rb | 44 +++++++------ test/cli/app_test.rb | 38 +++++------ test/commands/app_test.rb | 50 +++++++------- test/fixtures/deploy_with_accessories.yml | 2 +- 6 files changed, 126 insertions(+), 94 deletions(-) diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb index 796cd367..6e9cdad9 100644 --- a/lib/mrsk/cli/app.rb +++ b/lib/mrsk/cli/app.rb @@ -7,24 +7,26 @@ class Mrsk::Cli::App < Mrsk::Cli::Base cli = self - MRSK.config.roles.each do |role| - on(role.hosts) do |host| - execute *MRSK.auditor.record("Booted app version #{version}"), verbosity: :debug + on(MRSK.hosts) do |host| + roles = MRSK.roles_on(host) + + roles.each do |role| + execute *MRSK.auditor(role: role.name).record("Booted app version #{version}"), verbosity: :debug begin - old_version = capture_with_info(*MRSK.app.current_running_version).strip - execute *MRSK.app.run(role: role.name) + old_version = capture_with_info(*MRSK.app(role: role.name).current_running_version).strip + execute *MRSK.app(role: role.name).run sleep MRSK.config.readiness_delay - execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present? + execute *MRSK.app(role: role.name).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present? rescue SSHKit::Command::Failed => e if e.message =~ /already in use/ error "Rebooting container with same version #{version} already deployed on #{host} (may cause gap in zero-downtime promise!)" - execute *MRSK.auditor.record("Rebooted app version #{version}"), verbosity: :debug + execute *MRSK.auditor(role: role.name).record("Rebooted app version #{version}"), verbosity: :debug - execute *MRSK.app.stop(version: version) - execute *MRSK.app.remove_container(version: version) - execute *MRSK.app.run(role: role.name) + execute *MRSK.app(role: role.name).stop(version: version) + execute *MRSK.app(role: role.name).remove_container(version: version) + execute *MRSK.app(role: role.name).run else raise end @@ -36,24 +38,38 @@ class Mrsk::Cli::App < Mrsk::Cli::Base desc "start", "Start existing app container on servers" def start - on(MRSK.hosts) do - execute *MRSK.auditor.record("Started app version #{MRSK.version}"), verbosity: :debug - execute *MRSK.app.start, raise_on_non_zero_exit: false + on(MRSK.hosts) do |host| + roles = MRSK.roles_on(host) + + roles.each do |role| + execute *MRSK.auditor(role: role.name).record("Started app version #{MRSK.version}"), verbosity: :debug + execute *MRSK.app(role: role.name).start, raise_on_non_zero_exit: false + end end end desc "stop", "Stop app container on servers" def stop - on(MRSK.hosts) do - execute *MRSK.auditor.record("Stopped app"), verbosity: :debug - execute *MRSK.app.stop, raise_on_non_zero_exit: false + on(MRSK.hosts) do |host| + roles = MRSK.roles_on(host) + + roles.each do |role| + execute *MRSK.auditor(role: role.name).record("Stopped app"), verbosity: :debug + execute *MRSK.app(role: role.name).stop, raise_on_non_zero_exit: false + end end end # FIXME: Drop in favor of just containers? desc "details", "Show details about app containers" def details - on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.info) } + on(MRSK.hosts) do |host| + roles = MRSK.roles_on(host) + + roles.each do |role| + puts_by_host host, capture_with_info(*MRSK.app(role: role.name).info) + end + end end desc "exec [CMD]", "Execute a custom command on servers (use --help to show options)" @@ -65,7 +81,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base say "Get current version of running container...", :magenta unless options[:version] using_version(options[:version] || current_running_version) do |version| say "Launching interactive command with version #{version} via SSH from existing container on #{MRSK.primary_host}...", :magenta - run_locally { exec MRSK.app.execute_in_existing_container_over_ssh(cmd, host: MRSK.primary_host) } + run_locally { exec MRSK.app(role: role.name).execute_in_existing_container_over_ssh(cmd, host: MRSK.primary_host) } end when options[:interactive] @@ -81,8 +97,12 @@ class Mrsk::Cli::App < Mrsk::Cli::Base say "Launching command with version #{version} from existing container...", :magenta on(MRSK.hosts) do |host| - execute *MRSK.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug - puts_by_host host, capture_with_info(*MRSK.app.execute_in_existing_container(cmd)) + roles = MRSK.roles_on(host) + + roles.each do |role| + execute *MRSK.auditor(role: role.name).record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug + puts_by_host host, capture_with_info(*MRSK.app(role: role.name).execute_in_existing_container(cmd)) + end end end @@ -147,17 +167,25 @@ class Mrsk::Cli::App < Mrsk::Cli::Base desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true def remove_container(version) - on(MRSK.hosts) do - execute *MRSK.auditor.record("Removed app container with version #{version}"), verbosity: :debug - execute *MRSK.app.remove_container(version: version) + on(MRSK.hosts) do |host| + roles = MRSK.roles_on(host) + + roles.each do |role| + execute *MRSK.auditor(role: role.name).record("Removed app container with version #{version}"), verbosity: :debug + execute *MRSK.app(role: role.name).remove_container(version: version) + end end end desc "remove_containers", "Remove all app containers from servers", hide: true def remove_containers - on(MRSK.hosts) do - execute *MRSK.auditor.record("Removed all app containers"), verbosity: :debug - execute *MRSK.app.remove_containers + on(MRSK.hosts) do |host| + roles = MRSK.roles_on(host) + + roles.each do |role| + execute *MRSK.auditor(role: role.name).record("Removed all app containers"), verbosity: :debug + execute *MRSK.app(role: role.name).remove_containers + end end end diff --git a/lib/mrsk/commander.rb b/lib/mrsk/commander.rb index d49243c3..7dce1cf6 100644 --- a/lib/mrsk/commander.rb +++ b/lib/mrsk/commander.rb @@ -45,7 +45,7 @@ class Mrsk::Commander end def roles_on(host) - roles.select { |role| role.hosts.include?(host) } + roles.select { |role| role.hosts.include?(host.to_s) } end def traefik_hosts @@ -61,8 +61,8 @@ class Mrsk::Commander end - def app - @app ||= Mrsk::Commands::App.new(config) + def app(role: nil) + Mrsk::Commands::App.new(config, role: role) end def accessory(name) diff --git a/lib/mrsk/commands/app.rb b/lib/mrsk/commands/app.rb index 906a30d2..2008e8ff 100644 --- a/lib/mrsk/commands/app.rb +++ b/lib/mrsk/commands/app.rb @@ -1,12 +1,19 @@ class Mrsk::Commands::App < Mrsk::Commands::Base - def run(role: :web) - role = config.role(role) + attr_reader :role + + def initialize(config, role: nil) + super(config) + @role = role + end + + def run + role = config.role(self.role) docker :run, "--detach", "--restart unless-stopped", "--log-opt", "max-size=#{MAX_LOG_SIZE}", - "--name", service_with_version_and_destination, + "--name", service_with_version_and_destination_and_role, *role.env_args, *config.volume_args, *role.label_args, @@ -16,7 +23,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base end def start - docker :start, service_with_version_and_destination + docker :start, service_with_version_and_destination_and_role end def stop(version: nil) @@ -26,7 +33,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base end def info - docker :ps, *service_filter_with_destination + docker :ps, *service_filter_with_destination_and_role end @@ -51,7 +58,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base def execute_in_existing_container(*command, interactive: false) docker :exec, ("-it" if interactive), - service_with_version_and_destination, + service_with_version_and_destination_and_role, *command end @@ -75,13 +82,13 @@ class Mrsk::Commands::App < Mrsk::Commands::Base def current_container_id - docker :ps, "--quiet", *service_filter_with_destination + docker :ps, "--quiet", *service_filter_with_destination_and_role end def current_running_version # FIXME: Find more graceful way to extract the version from "app-version" than using sed and tail! pipe \ - docker(:ps, *service_filter_with_destination, "--format", '"{{.Names}}"'), + docker(:ps, *service_filter_with_destination_and_role, "--format", '"{{.Names}}"'), %(sed 's/-/\\n/g'), "tail -n 1" end @@ -100,7 +107,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base def list_containers - docker :container, :ls, "--all", *service_filter_with_destination + docker :container, :ls, "--all", *service_filter_with_destination_and_role # TODO: role hier needed oder sogar falsch? end def list_container_names @@ -109,12 +116,12 @@ class Mrsk::Commands::App < Mrsk::Commands::Base def remove_container(version:) pipe \ - container_id_for(container_name: service_with_version_and_destination(version)), + container_id_for(container_name: service_with_version_and_destination_and_role(version)), xargs(docker(:container, :rm)) end def remove_containers - docker :container, :prune, "--force", *service_filter_with_destination + docker :container, :prune, "--force", *service_filter_with_destination_and_role end def list_images @@ -127,23 +134,22 @@ class Mrsk::Commands::App < Mrsk::Commands::Base private - def service_with_version_and_destination(version = nil) - [ config.service, config.destination, version || config.version ].compact.join("-") + def service_with_version_and_destination_and_role(version = nil) + [ config.service, role, config.destination, version || config.version ].compact.join("-") # TODO: is role sometimes nil here? bis jetzt wars nie nil end def container_id_for_version(version) - container_id_for(container_name: service_with_version_and_destination(version)) + container_id_for(container_name: service_with_version_and_destination_and_role(version)) end def service_filter [ "--filter", "label=service=#{config.service}" ] end - def service_filter_with_destination - if config.destination - service_filter << "label=destination=#{config.destination}" - else - service_filter + def service_filter_with_destination_and_role + service_filter.tap do |filter| + filter << "label=destination=#{config.destination}" if config.destination + filter << "label=role=#{role}" if role end end end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 76f59a80..1c0899fe 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -9,8 +9,8 @@ class CliAppTest < CliTestCase .returns("123") # old version run_command("boot").tap do |output| - assert_match /docker run --detach --restart unless-stopped/, output - assert_match /docker container ls --all --filter name=app-123 --quiet | xargs docker stop/, output + assert_match %r[docker run --detach --restart unless-stopped], output + assert_match %r[docker container ls --all --filter name=app-web-123 --quiet | xargs docker stop], output end end @@ -20,18 +20,16 @@ class CliAppTest < CliTestCase # Prevent expected failures from outputting to terminal Thread.report_on_exception = false - MRSK.app.stubs(:run) - .raises(SSHKit::Command::Failed.new("already in use")) - .then + Mrsk::Commands::App.any_instance.stubs(:run) .raises(SSHKit::Command::Failed.new("already in use")) .then .returns([ :docker, :run ]) run_command("boot").tap do |output| - assert_match /Rebooting container with same version 999 already deployed/, output # Can't start what's already running - assert_match /docker container ls --all --filter name=app-999 --quiet | xargs docker container rm/, output # Stop old running - assert_match /docker container ls --all --filter name=app-999 --quiet | xargs docker container rm/, output # Remove old container - assert_match /docker run/, output # Start new container + assert_match %r[Rebooting container with same version 999 already deployed], output # Can't start what's already running + assert_match %r[docker container ls --all --filter name=app-web-999 --quiet | xargs docker stop], output # Stop old running + assert_match %r[docker container ls --all --filter name=app-web-999 --quiet | xargs docker container rm], output # Remove old container + assert_match %r[docker run on 1.1.1.1], output # Start new container end ensure Thread.report_on_exception = true @@ -39,51 +37,51 @@ class CliAppTest < CliTestCase test "start" do run_command("start").tap do |output| - assert_match /docker start app-999/, output + assert_match %r[docker start app-web-999], output end end test "stop" do run_command("stop").tap do |output| - assert_match /docker ps --quiet --filter label=service=app \| xargs docker stop/, output + assert_match %r[docker ps --quiet --filter label=service=app label=role=web | xargs docker stop], output end end test "details" do run_command("details").tap do |output| - assert_match /docker ps --filter label=service=app/, output + assert_match %r[docker ps --filter label=service=app label=role=web], output end end test "remove" do run_command("remove").tap do |output| - assert_match /docker ps --quiet --filter label=service=app | xargs docker stop/, output - assert_match /docker container prune --force --filter label=service=app/, output - assert_match /docker image prune --all --force --filter label=service=app/, output + assert_match %r[docker ps --quiet --filter label=service=app label=role=web | xargs docker stop], output + assert_match %r[docker container prune --force --filter label=service=app], output + assert_match %r[docker image prune --all --force --filter label=service=app], output end end test "remove_container" do run_command("remove_container", "1234567").tap do |output| - assert_match /docker container ls --all --filter name=app-1234567 --quiet \| xargs docker container rm/, output + assert_match %r[docker container ls --all --filter name=app-web-1234567 --quiet | xargs docker container rm], output end end test "exec" do run_command("exec", "ruby -v").tap do |output| - assert_match /ruby -v/, output + assert_match %r[docker run --rm dhh/app:999 ruby -v], output end end test "exec with reuse" do run_command("exec", "--reuse", "ruby -v").tap do |output| - assert_match %r[docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1], output # Get current version - assert_match %r[docker exec app-999 ruby -v], output + assert_match %r[docker ps --filter label=service=app --format "{{.Names}}" | sed 's/-/\n/g' | tail -n 1], output + assert_match %r[docker exec app-web-999 ruby -v], output end end private def run_command(*command) - stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) } + stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1"]) } end end diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index da436bcb..3f0655c3 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -5,7 +5,7 @@ class CommandsAppTest < ActiveSupport::TestCase ENV["RAILS_MASTER_KEY"] = "456" @config = { service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], env: { "secret" => [ "RAILS_MASTER_KEY" ] } } - @app = Mrsk::Commands::App.new Mrsk::Configuration.new(@config).tap { |c| c.version = "999" } + @app = Mrsk::Commands::App.new(Mrsk::Configuration.new(@config).tap { |c| c.version = "999" }, role: "web") end teardown do @@ -14,7 +14,7 @@ class CommandsAppTest < ActiveSupport::TestCase test "run" do assert_equal \ - "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-999 -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app.retry.attempts=\"5\" --label traefik.http.middlewares.app.retry.initialinterval=\"500ms\" dhh/app:999", + "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-web-999 -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app.retry.attempts=\"5\" --label traefik.http.middlewares.app.retry.initialinterval=\"500ms\" dhh/app:999", @app.run.join(" ") end @@ -22,7 +22,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:volumes] = ["/local/path:/container/path" ] assert_equal \ - "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-999 -e RAILS_MASTER_KEY=\"456\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app.retry.attempts=\"5\" --label traefik.http.middlewares.app.retry.initialinterval=\"500ms\" dhh/app:999", + "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-web-999 -e RAILS_MASTER_KEY=\"456\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app.retry.attempts=\"5\" --label traefik.http.middlewares.app.retry.initialinterval=\"500ms\" dhh/app:999", @app.run.join(" ") end @@ -30,72 +30,72 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:healthcheck] = { "path" => "/healthz" } assert_equal \ - "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-999 -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/healthz\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app.retry.attempts=\"5\" --label traefik.http.middlewares.app.retry.initialinterval=\"500ms\" dhh/app:999", + "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-web-999 -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/healthz\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app.retry.attempts=\"5\" --label traefik.http.middlewares.app.retry.initialinterval=\"500ms\" dhh/app:999", @app.run.join(" ") end test "run with custom options" do @config[:servers] = { "web" => [ "1.1.1.1" ], "jobs" => { "hosts" => [ "1.1.1.2" ], "cmd" => "bin/jobs", "options" => { "mount" => "somewhere", "cap-add" => true } } } - @app = Mrsk::Commands::App.new Mrsk::Configuration.new(@config).tap { |c| c.version = "999" } + @app = Mrsk::Commands::App.new(Mrsk::Configuration.new(@config).tap { |c| c.version = "999" }, role: "jobs") assert_equal \ - "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-999 -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs", - @app.run(role: :jobs).join(" ") + "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-jobs-999 -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs", + @app.run.join(" ") end test "start" do assert_equal \ - "docker start app-999", + "docker start app-web-999", @app.start.join(" ") end test "stop" do assert_equal \ - "docker ps --quiet --filter label=service=app | xargs docker stop", + "docker ps --quiet --filter label=service=app label=role=web | xargs docker stop", @app.stop.join(" ") end test "info" do assert_equal \ - "docker ps --filter label=service=app", + "docker ps --filter label=service=app label=role=web", @app.info.join(" ") end test "logs" do assert_equal \ - "docker ps --quiet --filter label=service=app | xargs docker logs 2>&1", + "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs 2>&1", @app.logs.join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app | xargs docker logs --since 5m 2>&1", + "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs --since 5m 2>&1", @app.logs(since: "5m").join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app | xargs docker logs --tail 100 2>&1", + "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs --tail 100 2>&1", @app.logs(lines: "100").join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app | xargs docker logs --since 5m --tail 100 2>&1", + "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs --since 5m --tail 100 2>&1", @app.logs(since: "5m", lines: "100").join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app | xargs docker logs 2>&1 | grep 'my-id'", + "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs 2>&1 | grep 'my-id'", @app.logs(grep: "my-id").join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app | xargs docker logs --since 5m 2>&1 | grep 'my-id'", + "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs --since 5m 2>&1 | grep 'my-id'", @app.logs(since: "5m", grep: "my-id").join(" ") end test "follow logs" do @app.stub(:run_over_ssh, ->(cmd, host:) { cmd.join(" ") }) do assert_equal \ - "docker ps --quiet --filter label=service=app | xargs docker logs --timestamps --tail 10 --follow 2>&1", + "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1", @app.follow_logs(host: "app-1") assert_equal \ - "docker ps --quiet --filter label=service=app | xargs docker logs --timestamps --tail 10 --follow 2>&1 | grep \"Completed\"", + "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1 | grep \"Completed\"", @app.follow_logs(host: "app-1", grep: "Completed") end end @@ -109,20 +109,20 @@ class CommandsAppTest < ActiveSupport::TestCase test "execute in existing container" do assert_equal \ - "docker exec app-999 bin/rails db:setup", + "docker exec app-web-999 bin/rails db:setup", @app.execute_in_existing_container("bin/rails", "db:setup").join(" ") end test "execute in new container over ssh" do @app.stub(:run_over_ssh, ->(cmd, host:) { cmd.join(" ") }) do - assert_match %r|docker run -it --rm -e RAILS_MASTER_KEY=\"456\" dhh/app:999 bin/rails c|, + assert_match %r[docker run -it --rm -e RAILS_MASTER_KEY=\"456\" dhh/app:999 bin/rails c], @app.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1") end end test "execute in existing container over ssh" do @app.stub(:run_over_ssh, ->(cmd, host:) { cmd.join(" ") }) do - assert_match %r|docker exec -it app-999 bin/rails c|, + assert_match %r[docker exec -it app-web-999 bin/rails c], @app.execute_in_existing_container_over_ssh("bin/rails", "c", host: "app-1") end end @@ -154,19 +154,19 @@ class CommandsAppTest < ActiveSupport::TestCase test "current_container_id" do assert_equal \ - "docker ps --quiet --filter label=service=app", + "docker ps --quiet --filter label=service=app label=role=web", @app.current_container_id.join(" ") end test "container_id_for" do assert_equal \ - "docker container ls --all --filter name=app-999 --quiet", - @app.container_id_for(container_name: "app-999").join(" ") + "docker container ls --all --filter name=app-web-999 --quiet", + @app.container_id_for(container_name: "app-web-999").join(" ") end test "current_running_version" do assert_equal \ - "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", + "docker ps --filter label=service=app label=role=web --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", @app.current_running_version.join(" ") end diff --git a/test/fixtures/deploy_with_accessories.yml b/test/fixtures/deploy_with_accessories.yml index 0d0c86f3..4440a1f9 100644 --- a/test/fixtures/deploy_with_accessories.yml +++ b/test/fixtures/deploy_with_accessories.yml @@ -28,4 +28,4 @@ accessories: directories: - data:/data -readiness_delay: 0 \ No newline at end of file +readiness_delay: 0 From 7d4dfc4c868711a3d14ea2bb874f90c272a80e1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20B=C3=BChlmann?= Date: Fri, 10 Mar 2023 09:18:47 +0100 Subject: [PATCH 05/32] Pass role names for simplicity --- lib/mrsk/cli/app.rb | 40 ++++++++++++++++++++-------------------- lib/mrsk/commander.rb | 2 +- test/commander_test.rb | 6 +++--- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb index 6e9cdad9..8f6cd5f4 100644 --- a/lib/mrsk/cli/app.rb +++ b/lib/mrsk/cli/app.rb @@ -11,22 +11,22 @@ class Mrsk::Cli::App < Mrsk::Cli::Base roles = MRSK.roles_on(host) roles.each do |role| - execute *MRSK.auditor(role: role.name).record("Booted app version #{version}"), verbosity: :debug + execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug begin - old_version = capture_with_info(*MRSK.app(role: role.name).current_running_version).strip - execute *MRSK.app(role: role.name).run + old_version = capture_with_info(*MRSK.app(role: role).current_running_version).strip + execute *MRSK.app(role: role).run sleep MRSK.config.readiness_delay - execute *MRSK.app(role: role.name).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present? + execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present? rescue SSHKit::Command::Failed => e if e.message =~ /already in use/ error "Rebooting container with same version #{version} already deployed on #{host} (may cause gap in zero-downtime promise!)" - execute *MRSK.auditor(role: role.name).record("Rebooted app version #{version}"), verbosity: :debug + execute *MRSK.auditor(role: role).record("Rebooted app version #{version}"), verbosity: :debug - execute *MRSK.app(role: role.name).stop(version: version) - execute *MRSK.app(role: role.name).remove_container(version: version) - execute *MRSK.app(role: role.name).run + execute *MRSK.app(role: role).stop(version: version) + execute *MRSK.app(role: role).remove_container(version: version) + execute *MRSK.app(role: role).run else raise end @@ -42,8 +42,8 @@ class Mrsk::Cli::App < Mrsk::Cli::Base roles = MRSK.roles_on(host) roles.each do |role| - execute *MRSK.auditor(role: role.name).record("Started app version #{MRSK.version}"), verbosity: :debug - execute *MRSK.app(role: role.name).start, raise_on_non_zero_exit: false + execute *MRSK.auditor(role: role).record("Started app version #{MRSK.version}"), verbosity: :debug + execute *MRSK.app(role: role).start, raise_on_non_zero_exit: false end end end @@ -54,8 +54,8 @@ class Mrsk::Cli::App < Mrsk::Cli::Base roles = MRSK.roles_on(host) roles.each do |role| - execute *MRSK.auditor(role: role.name).record("Stopped app"), verbosity: :debug - execute *MRSK.app(role: role.name).stop, raise_on_non_zero_exit: false + execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug + execute *MRSK.app(role: role).stop, raise_on_non_zero_exit: false end end end @@ -67,7 +67,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base roles = MRSK.roles_on(host) roles.each do |role| - puts_by_host host, capture_with_info(*MRSK.app(role: role.name).info) + puts_by_host host, capture_with_info(*MRSK.app(role: role).info) end end end @@ -81,7 +81,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base say "Get current version of running container...", :magenta unless options[:version] using_version(options[:version] || current_running_version) do |version| say "Launching interactive command with version #{version} via SSH from existing container on #{MRSK.primary_host}...", :magenta - run_locally { exec MRSK.app(role: role.name).execute_in_existing_container_over_ssh(cmd, host: MRSK.primary_host) } + run_locally { exec MRSK.app(role: role).execute_in_existing_container_over_ssh(cmd, host: MRSK.primary_host) } end when options[:interactive] @@ -100,8 +100,8 @@ class Mrsk::Cli::App < Mrsk::Cli::Base roles = MRSK.roles_on(host) roles.each do |role| - execute *MRSK.auditor(role: role.name).record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug - puts_by_host host, capture_with_info(*MRSK.app(role: role.name).execute_in_existing_container(cmd)) + execute *MRSK.auditor(role: role).record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug + puts_by_host host, capture_with_info(*MRSK.app(role: role).execute_in_existing_container(cmd)) end end end @@ -171,8 +171,8 @@ class Mrsk::Cli::App < Mrsk::Cli::Base roles = MRSK.roles_on(host) roles.each do |role| - execute *MRSK.auditor(role: role.name).record("Removed app container with version #{version}"), verbosity: :debug - execute *MRSK.app(role: role.name).remove_container(version: version) + execute *MRSK.auditor(role: role).record("Removed app container with version #{version}"), verbosity: :debug + execute *MRSK.app(role: role).remove_container(version: version) end end end @@ -183,8 +183,8 @@ class Mrsk::Cli::App < Mrsk::Cli::Base roles = MRSK.roles_on(host) roles.each do |role| - execute *MRSK.auditor(role: role.name).record("Removed all app containers"), verbosity: :debug - execute *MRSK.app(role: role.name).remove_containers + execute *MRSK.auditor(role: role).record("Removed all app containers"), verbosity: :debug + execute *MRSK.app(role: role).remove_containers end end end diff --git a/lib/mrsk/commander.rb b/lib/mrsk/commander.rb index 7dce1cf6..a4edb48a 100644 --- a/lib/mrsk/commander.rb +++ b/lib/mrsk/commander.rb @@ -45,7 +45,7 @@ class Mrsk::Commander end def roles_on(host) - roles.select { |role| role.hosts.include?(host.to_s) } + roles.select { |role| role.hosts.include?(host.to_s) }.map(&:name) end def traefik_hosts diff --git a/test/commander_test.rb b/test/commander_test.rb index 918f69f8..ad06d566 100644 --- a/test/commander_test.rb +++ b/test/commander_test.rb @@ -60,8 +60,8 @@ class CommanderTest < ActiveSupport::TestCase end test "roles_on" do - assert_equal [ "web", "workers" ], @mrsk.roles_on("1.1.1.1").map(&:name) - assert_equal [ "web" ], @mrsk.roles_on("1.1.1.2").map(&:name) - assert_equal [ "workers" ], @mrsk.roles_on("1.1.1.3").map(&:name) + assert_equal [ "web", "workers" ], @mrsk.roles_on("1.1.1.1") + assert_equal [ "web" ], @mrsk.roles_on("1.1.1.2") + assert_equal [ "workers" ], @mrsk.roles_on("1.1.1.3") end end From 418bc13ae74c89dda09bade531de5006fef01f20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20B=C3=BChlmann?= Date: Fri, 10 Mar 2023 10:33:55 +0100 Subject: [PATCH 06/32] Apply filters correctly --- lib/mrsk/commands/app.rb | 4 ++-- test/cli/app_test.rb | 32 ++++++++++++++++---------------- test/commands/app_test.rb | 24 ++++++++++++------------ 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/lib/mrsk/commands/app.rb b/lib/mrsk/commands/app.rb index 2008e8ff..721c8495 100644 --- a/lib/mrsk/commands/app.rb +++ b/lib/mrsk/commands/app.rb @@ -148,8 +148,8 @@ class Mrsk::Commands::App < Mrsk::Commands::Base def service_filter_with_destination_and_role service_filter.tap do |filter| - filter << "label=destination=#{config.destination}" if config.destination - filter << "label=role=#{role}" if role + filter.concat [ "--filter", "label=destination=#{config.destination}" ] if config.destination + filter.concat [ "--filter", "label=role=#{role}" ] if role end end end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 1c0899fe..4cc2ae08 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -9,8 +9,8 @@ class CliAppTest < CliTestCase .returns("123") # old version run_command("boot").tap do |output| - assert_match %r[docker run --detach --restart unless-stopped], output - assert_match %r[docker container ls --all --filter name=app-web-123 --quiet | xargs docker stop], output + assert_match /#{Regexp.escape("docker run --detach --restart unless-stopped")}/, output + assert_match /#{Regexp.escape("docker container ls --all --filter name=app-web-123 --quiet | xargs docker stop")}/, output end end @@ -26,10 +26,10 @@ class CliAppTest < CliTestCase .returns([ :docker, :run ]) run_command("boot").tap do |output| - assert_match %r[Rebooting container with same version 999 already deployed], output # Can't start what's already running - assert_match %r[docker container ls --all --filter name=app-web-999 --quiet | xargs docker stop], output # Stop old running - assert_match %r[docker container ls --all --filter name=app-web-999 --quiet | xargs docker container rm], output # Remove old container - assert_match %r[docker run on 1.1.1.1], output # Start new container + assert_match /#{Regexp.escape("Rebooting container with same version 999 already deployed")}/, output # Can't start what's already running + assert_match /#{Regexp.escape("docker container ls --all --filter name=app-web-999 --quiet | xargs docker stop")}/, output # Stop old running + assert_match /#{Regexp.escape("docker container ls --all --filter name=app-web-999 --quiet | xargs docker container rm")}/, output # Remove old container + assert_match /#{Regexp.escape("docker run on 1.1.1.1")}/, output # Start new container end ensure Thread.report_on_exception = true @@ -37,46 +37,46 @@ class CliAppTest < CliTestCase test "start" do run_command("start").tap do |output| - assert_match %r[docker start app-web-999], output + assert_match /#{Regexp.escape("docker start app-web-999")}/, output end end test "stop" do run_command("stop").tap do |output| - assert_match %r[docker ps --quiet --filter label=service=app label=role=web | xargs docker stop], output + assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop")}/, output end end test "details" do run_command("details").tap do |output| - assert_match %r[docker ps --filter label=service=app label=role=web], output + assert_match /#{Regexp.escape("docker ps --filter label=service=app --filter label=role=web")}/, output end end test "remove" do run_command("remove").tap do |output| - assert_match %r[docker ps --quiet --filter label=service=app label=role=web | xargs docker stop], output - assert_match %r[docker container prune --force --filter label=service=app], output - assert_match %r[docker image prune --all --force --filter label=service=app], output + assert_match /#{Regexp.escape("docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop")}/, output + assert_match /#{Regexp.escape("docker container prune --force --filter label=service=app")}/, output + assert_match /#{Regexp.escape("docker image prune --all --force --filter label=service=app")}/, output end end test "remove_container" do run_command("remove_container", "1234567").tap do |output| - assert_match %r[docker container ls --all --filter name=app-web-1234567 --quiet | xargs docker container rm], output + assert_match /#{Regexp.escape("docker container ls --all --filter name=app-web-1234567 --quiet | xargs docker container rm")}/, output end end test "exec" do run_command("exec", "ruby -v").tap do |output| - assert_match %r[docker run --rm dhh/app:999 ruby -v], output + assert_match /#{Regexp.escape("docker run --rm dhh/app:999 ruby -v")}/, output end end test "exec with reuse" do run_command("exec", "--reuse", "ruby -v").tap do |output| - assert_match %r[docker ps --filter label=service=app --format "{{.Names}}" | sed 's/-/\n/g' | tail -n 1], output - assert_match %r[docker exec app-web-999 ruby -v], output + assert_match /#{Regexp.escape("docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1")}/, output + assert_match /#{Regexp.escape("docker exec app-web-999 ruby -v")}/, output end end diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index 3f0655c3..ce485c44 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -51,51 +51,51 @@ class CommandsAppTest < ActiveSupport::TestCase test "stop" do assert_equal \ - "docker ps --quiet --filter label=service=app label=role=web | xargs docker stop", + "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop", @app.stop.join(" ") end test "info" do assert_equal \ - "docker ps --filter label=service=app label=role=web", + "docker ps --filter label=service=app --filter label=role=web", @app.info.join(" ") end test "logs" do assert_equal \ - "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs 2>&1", + "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs 2>&1", @app.logs.join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs --since 5m 2>&1", + "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m 2>&1", @app.logs(since: "5m").join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs --tail 100 2>&1", + "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --tail 100 2>&1", @app.logs(lines: "100").join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs --since 5m --tail 100 2>&1", + "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m --tail 100 2>&1", @app.logs(since: "5m", lines: "100").join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs 2>&1 | grep 'my-id'", + "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs 2>&1 | grep 'my-id'", @app.logs(grep: "my-id").join(" ") assert_equal \ - "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs --since 5m 2>&1 | grep 'my-id'", + "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --since 5m 2>&1 | grep 'my-id'", @app.logs(since: "5m", grep: "my-id").join(" ") end test "follow logs" do @app.stub(:run_over_ssh, ->(cmd, host:) { cmd.join(" ") }) do assert_equal \ - "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1", + "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1", @app.follow_logs(host: "app-1") assert_equal \ - "docker ps --quiet --filter label=service=app label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1 | grep \"Completed\"", + "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker logs --timestamps --tail 10 --follow 2>&1 | grep \"Completed\"", @app.follow_logs(host: "app-1", grep: "Completed") end end @@ -154,7 +154,7 @@ class CommandsAppTest < ActiveSupport::TestCase test "current_container_id" do assert_equal \ - "docker ps --quiet --filter label=service=app label=role=web", + "docker ps --quiet --filter label=service=app --filter label=role=web", @app.current_container_id.join(" ") end @@ -166,7 +166,7 @@ class CommandsAppTest < ActiveSupport::TestCase test "current_running_version" do assert_equal \ - "docker ps --filter label=service=app label=role=web --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", + "docker ps --filter label=service=app --filter label=role=web --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", @app.current_running_version.join(" ") end From ccf8762c98b939659d643179a668ab07b16b36fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20B=C3=BChlmann?= Date: Fri, 10 Mar 2023 10:50:26 +0100 Subject: [PATCH 07/32] Reuse web container per default --- lib/mrsk/cli/app.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb index 8f6cd5f4..bb9554bb 100644 --- a/lib/mrsk/cli/app.rb +++ b/lib/mrsk/cli/app.rb @@ -81,7 +81,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base say "Get current version of running container...", :magenta unless options[:version] using_version(options[:version] || current_running_version) do |version| say "Launching interactive command with version #{version} via SSH from existing container on #{MRSK.primary_host}...", :magenta - run_locally { exec MRSK.app(role: role).execute_in_existing_container_over_ssh(cmd, host: MRSK.primary_host) } + run_locally { exec MRSK.app(role: "web").execute_in_existing_container_over_ssh(cmd, host: MRSK.primary_host) } end when options[:interactive] From 72e0184e9f6350f966cf8aa7ea6649e4bee52d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20B=C3=BChlmann?= Date: Mon, 13 Mar 2023 17:36:02 +0100 Subject: [PATCH 08/32] Fix failing tests --- lib/mrsk/commands/app.rb | 2 +- test/commands/app_test.rb | 2 +- test/commands/healthcheck_test.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/mrsk/commands/app.rb b/lib/mrsk/commands/app.rb index e20fc08e..8d2bad6f 100644 --- a/lib/mrsk/commands/app.rb +++ b/lib/mrsk/commands/app.rb @@ -14,7 +14,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base "--restart unless-stopped", "--log-opt", "max-size=#{MAX_LOG_SIZE}", "--name", service_with_version_and_destination_and_role, - "-e", "MRSK_CONTAINER_NAME=\"#{service_with_version_and_destination}\"", + "-e", "MRSK_CONTAINER_NAME=\"#{service_with_version_and_destination_and_role}\"", *role.env_args, *config.volume_args, *role.label_args, diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index 34039820..80592ea6 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -40,7 +40,7 @@ class CommandsAppTest < ActiveSupport::TestCase assert_equal \ "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-jobs-999 -e MRSK_CONTAINER_NAME=\"app-jobs-999\" -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs", - @app.run(role: :jobs).join(" ") + @app.run.join(" ") end test "start" do diff --git a/test/commands/healthcheck_test.rb b/test/commands/healthcheck_test.rb index cd1f0d2d..effd3ef3 100644 --- a/test/commands/healthcheck_test.rb +++ b/test/commands/healthcheck_test.rb @@ -33,7 +33,7 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase test "run with custom options" do @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere" } } } assert_equal \ - "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app --mount \"somewhere\" dhh/app:123", + "docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e MRSK_CONTAINER_NAME=\"healthcheck-app\" --mount \"somewhere\" dhh/app:123", new_command.run.join(" ") end From a15603655c9200e39f864894d42b44074b6cc01c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20B=C3=BChlmann?= Date: Wed, 15 Mar 2023 09:28:10 +0100 Subject: [PATCH 09/32] Adapt test for single host --- test/cli/main_test.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index f72741dd..33e4629d 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -108,8 +108,6 @@ class CliMainTest < CliTestCase run_command("audit").tap do |output| assert_match /tail -n 50 mrsk-app-audit.log on 1.1.1.1/, output assert_match /App Host: 1.1.1.1/, output - assert_match /tail -n 50 mrsk-app-audit.log on 1.1.1.2/, output - assert_match /App Host: 1.1.1.2/, output end end From b5372988f73c9e2763d3a0056dcc82c5fe24cf1e Mon Sep 17 00:00:00 2001 From: Samuel Sieg Date: Sun, 19 Mar 2023 09:21:08 +0100 Subject: [PATCH 10/32] Add global logging configuration --- lib/mrsk/commands/accessory.rb | 2 +- lib/mrsk/commands/app.rb | 2 +- lib/mrsk/commands/base.rb | 2 -- lib/mrsk/commands/traefik.rb | 2 +- lib/mrsk/configuration.rb | 12 ++++++- test/cli/accessory_test.rb | 6 ++-- test/cli/traefik_test.rb | 2 +- test/commands/accessory_test.rb | 57 ++++++++++++++++++++------------- test/commands/app_test.rb | 16 ++++++--- test/commands/traefik_test.rb | 14 ++++++-- test/configuration_test.rb | 9 ++++-- 11 files changed, 83 insertions(+), 41 deletions(-) diff --git a/lib/mrsk/commands/accessory.rb b/lib/mrsk/commands/accessory.rb index dbffc71b..adad2b43 100644 --- a/lib/mrsk/commands/accessory.rb +++ b/lib/mrsk/commands/accessory.rb @@ -12,8 +12,8 @@ class Mrsk::Commands::Accessory < Mrsk::Commands::Base "--name", service_name, "--detach", "--restart", "unless-stopped", - "--log-opt", "max-size=#{MAX_LOG_SIZE}", "--publish", port, + *config.logging_args, *env_args, *volume_args, *label_args, diff --git a/lib/mrsk/commands/app.rb b/lib/mrsk/commands/app.rb index 37d70476..1b8f36f6 100644 --- a/lib/mrsk/commands/app.rb +++ b/lib/mrsk/commands/app.rb @@ -5,10 +5,10 @@ class Mrsk::Commands::App < Mrsk::Commands::Base docker :run, "--detach", "--restart unless-stopped", - "--log-opt", "max-size=#{MAX_LOG_SIZE}", "--name", service_with_version_and_destination, "-e", "MRSK_CONTAINER_NAME=\"#{service_with_version_and_destination}\"", *role.env_args, + *config.logging_args, *config.volume_args, *role.label_args, *role.option_args, diff --git a/lib/mrsk/commands/base.rb b/lib/mrsk/commands/base.rb index e453dddc..76994ed6 100644 --- a/lib/mrsk/commands/base.rb +++ b/lib/mrsk/commands/base.rb @@ -2,8 +2,6 @@ module Mrsk::Commands class Base delegate :redact, to: Mrsk::Utils - MAX_LOG_SIZE = "10m" - attr_accessor :config def initialize(config) diff --git a/lib/mrsk/commands/traefik.rb b/lib/mrsk/commands/traefik.rb index b9ff19c1..1f6b8cc4 100644 --- a/lib/mrsk/commands/traefik.rb +++ b/lib/mrsk/commands/traefik.rb @@ -7,9 +7,9 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base docker :run, "--name traefik", "--detach", "--restart", "unless-stopped", - "--log-opt", "max-size=#{MAX_LOG_SIZE}", "--publish", port, "--volume", "/var/run/docker.sock:/var/run/docker.sock", + *config.logging_args, "traefik", "--providers.docker", "--log.level=DEBUG", diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index 03413116..7dc6f139 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -6,7 +6,7 @@ require "erb" require "net/ssh/proxy/jump" class Mrsk::Configuration - delegate :service, :image, :servers, :env, :labels, :registry, :builder, to: :raw_config, allow_nil: true + delegate :service, :image, :servers, :env, :labels, :registry, :builder, :logging, to: :raw_config, allow_nil: true delegate :argumentize, :argumentize_env_with_secrets, to: Mrsk::Utils attr_accessor :version @@ -109,6 +109,15 @@ class Mrsk::Configuration end end + def logging_args + if raw_config.logging.present? + [ "--log-driver", raw_config.logging["driver"] || "json-file" ] + + argumentize("--log-opt", raw_config.logging["options"]) + else + [] + end + end + def ssh_user if raw_config.ssh.present? @@ -161,6 +170,7 @@ class Mrsk::Configuration ssh_options: ssh_options, builder: raw_config.builder, accessories: raw_config.accessories, + logging: raw_config.logging, healthcheck: healthcheck }.compact end diff --git a/test/cli/accessory_test.rb b/test/cli/accessory_test.rb index fc6c25fb..891cd67a 100644 --- a/test/cli/accessory_test.rb +++ b/test/cli/accessory_test.rb @@ -6,7 +6,7 @@ class CliAccessoryTest < CliTestCase Mrsk::Cli::Accessory.any_instance.expects(:upload).with("mysql") run_command("boot", "mysql").tap do |output| - assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=10m --publish 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --publish 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output end end @@ -17,8 +17,8 @@ class CliAccessoryTest < CliTestCase Mrsk::Cli::Accessory.any_instance.expects(:upload).with("redis") run_command("boot", "all").tap do |output| - assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=10m --publish 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output - assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=10m --publish 6379:6379 --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.4", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --publish 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "docker run --name app-redis --detach --restart unless-stopped --publish 6379:6379 --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.4", output end end diff --git a/test/cli/traefik_test.rb b/test/cli/traefik_test.rb index 7eda6019..e1d86dee 100644 --- a/test/cli/traefik_test.rb +++ b/test/cli/traefik_test.rb @@ -3,7 +3,7 @@ require_relative "cli_test_case" class CliTraefikTest < CliTestCase test "boot" do run_command("boot").tap do |output| - assert_match "docker run --name traefik --detach --restart unless-stopped --log-opt max-size=10m --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG", output + assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG", output end end diff --git a/test/commands/accessory_test.rb b/test/commands/accessory_test.rb index c98e1590..9f5ca67b 100644 --- a/test/commands/accessory_test.rb +++ b/test/commands/accessory_test.rb @@ -36,10 +36,6 @@ class CommandsAccessoryTest < ActiveSupport::TestCase } } - @config = Mrsk::Configuration.new(@config) - @mysql = Mrsk::Commands::Accessory.new(@config, name: :mysql) - @redis = Mrsk::Commands::Accessory.new(@config, name: :redis) - ENV["MYSQL_ROOT_PASSWORD"] = "secret123" end @@ -49,56 +45,68 @@ class CommandsAccessoryTest < ActiveSupport::TestCase test "run" do assert_equal \ - "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=10m --publish 3306:3306 -e MYSQL_ROOT_PASSWORD=\"secret123\" -e MYSQL_ROOT_HOST=\"%\" --label service=\"app-mysql\" mysql:8.0", - @mysql.run.join(" ") + "docker run --name app-mysql --detach --restart unless-stopped --publish 3306:3306 -e MYSQL_ROOT_PASSWORD=\"secret123\" -e MYSQL_ROOT_HOST=\"%\" --label service=\"app-mysql\" mysql:8.0", + new_command(:mysql).run.join(" ") assert_equal \ - "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=10m --publish 6379:6379 -e SOMETHING=\"else\" --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest", - @redis.run.join(" ") + "docker run --name app-redis --detach --restart unless-stopped --publish 6379:6379 -e SOMETHING=\"else\" --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest", + new_command(:redis).run.join(" ") + end + + test "run with logging config" do + @config[:logging] = { "driver" => "local", "options" => { "max-size" => "10m", "max-file" => "3" } } + + assert_equal \ + "docker run --name app-mysql --detach --restart unless-stopped --publish 3306:3306 --log-driver local --log-opt max-size=\"10m\" --log-opt max-file=\"3\" -e MYSQL_ROOT_PASSWORD=\"secret123\" -e MYSQL_ROOT_HOST=\"%\" --label service=\"app-mysql\" mysql:8.0", + new_command(:mysql).run.join(" ") + + assert_equal \ + "docker run --name app-redis --detach --restart unless-stopped --publish 6379:6379 --log-driver local --log-opt max-size=\"10m\" --log-opt max-file=\"3\" -e SOMETHING=\"else\" --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest", + new_command(:redis).run.join(" ") end test "start" do assert_equal \ "docker container start app-mysql", - @mysql.start.join(" ") + new_command(:mysql).start.join(" ") end test "stop" do assert_equal \ "docker container stop app-mysql", - @mysql.stop.join(" ") + new_command(:mysql).stop.join(" ") end test "info" do assert_equal \ "docker ps --filter label=service=app-mysql", - @mysql.info.join(" ") + new_command(:mysql).info.join(" ") end test "execute in new container" do assert_equal \ "docker run --rm -e MYSQL_ROOT_PASSWORD=\"secret123\" -e MYSQL_ROOT_HOST=\"%\" mysql:8.0 mysql -u root", - @mysql.execute_in_new_container("mysql", "-u", "root").join(" ") + new_command(:mysql).execute_in_new_container("mysql", "-u", "root").join(" ") end test "execute in existing container" do assert_equal \ "docker exec app-mysql mysql -u root", - @mysql.execute_in_existing_container("mysql", "-u", "root").join(" ") + new_command(:mysql).execute_in_existing_container("mysql", "-u", "root").join(" ") end test "execute in new container over ssh" do - @mysql.stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do + new_command(:mysql).stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do assert_match %r|docker run -it --rm -e MYSQL_ROOT_PASSWORD=\"secret123\" -e MYSQL_ROOT_HOST=\"%\" mysql:8.0 mysql -u root|, - @mysql.execute_in_new_container_over_ssh("mysql", "-u", "root") + new_command(:mysql).execute_in_new_container_over_ssh("mysql", "-u", "root") end end test "execute in existing container over ssh" do - @mysql.stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do + new_command(:mysql).stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do assert_match %r|docker exec -it app-mysql mysql -u root|, - @mysql.execute_in_existing_container_over_ssh("mysql", "-u", "root") + new_command(:mysql).execute_in_existing_container_over_ssh("mysql", "-u", "root") end end @@ -107,28 +115,33 @@ class CommandsAccessoryTest < ActiveSupport::TestCase test "logs" do assert_equal \ "docker logs app-mysql --timestamps 2>&1", - @mysql.logs.join(" ") + new_command(:mysql).logs.join(" ") assert_equal \ "docker logs app-mysql --since 5m --tail 100 --timestamps 2>&1 | grep 'thing'", - @mysql.logs(since: "5m", lines: 100, grep: "thing").join(" ") + new_command(:mysql).logs(since: "5m", lines: 100, grep: "thing").join(" ") end test "follow logs" do assert_equal \ "ssh -t root@1.1.1.5 'docker logs app-mysql --timestamps --tail 10 --follow 2>&1'", - @mysql.follow_logs + new_command(:mysql).follow_logs end test "remove container" do assert_equal \ "docker container prune --force --filter label=service=app-mysql", - @mysql.remove_container.join(" ") + new_command(:mysql).remove_container.join(" ") end test "remove image" do assert_equal \ "docker image prune --all --force --filter label=service=app-mysql", - @mysql.remove_image.join(" ") + new_command(:mysql).remove_image.join(" ") end + + private + def new_command(accessory) + Mrsk::Commands::Accessory.new(Mrsk::Configuration.new(@config), name: accessory) + end end diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index c1a30f46..b47e46c2 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -14,7 +14,7 @@ class CommandsAppTest < ActiveSupport::TestCase test "run" do assert_equal \ - "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", @app.run.join(" ") end @@ -22,7 +22,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:volumes] = ["/local/path:/container/path" ] assert_equal \ - "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", @app.run.join(" ") end @@ -30,7 +30,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:healthcheck] = { "path" => "/healthz" } assert_equal \ - "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/healthz\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/healthz\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", @app.run.join(" ") end @@ -39,10 +39,18 @@ class CommandsAppTest < ActiveSupport::TestCase @app = Mrsk::Commands::App.new Mrsk::Configuration.new(@config).tap { |c| c.version = "999" } assert_equal \ - "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs", + "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs", @app.run(role: :jobs).join(" ") end + test "run with logging config" do + @config[:logging] = { "driver" => "local", "options" => { "max-size" => "10m", "max-file" => "3" } } + + assert_equal \ + "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --log-driver local --log-opt max-size=\"10m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", + @app.run.join(" ") + end + test "start" do assert_equal \ "docker start app-999", diff --git a/test/commands/traefik_test.rb b/test/commands/traefik_test.rb index 5ca51e13..9fa8b70f 100644 --- a/test/commands/traefik_test.rb +++ b/test/commands/traefik_test.rb @@ -10,12 +10,12 @@ class CommandsTraefikTest < ActiveSupport::TestCase test "run" do assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --log-opt max-size=10m --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", + "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") @config[:traefik]["host_port"] = "8080" assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --log-opt max-size=10m --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", + "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") end @@ -23,7 +23,15 @@ class CommandsTraefikTest < ActiveSupport::TestCase @config.delete(:traefik) assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --log-opt max-size=10m --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG", + "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG", + new_command.run.join(" ") + end + + test "run with logging config" do + @config[:logging] = { "driver" => "local", "options" => { "max-size" => "10m", "max-file" => "3" } } + + assert_equal \ + "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-driver local --log-opt max-size=\"10m\" --log-opt max-file=\"3\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") end diff --git a/test/configuration_test.rb b/test/configuration_test.rb index 5cd7856b..8e3fad42 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -9,7 +9,8 @@ class ConfigurationTest < ActiveSupport::TestCase registry: { "username" => "dhh", "password" => "secret" }, env: { "REDIS_URL" => "redis://x/y" }, servers: [ "1.1.1.1", "1.1.1.2" ], - volumes: ["/local/path:/container/path"] + volumes: ["/local/path:/container/path"], + logging: { "driver" => "json-file", "options" => { "max-size" => "10m" } } } @config = Mrsk::Configuration.new(@deploy) @@ -157,6 +158,10 @@ class ConfigurationTest < ActiveSupport::TestCase assert_equal ["--volume", "/local/path:/container/path"], @config.volume_args end + test "logging_args" do + assert_equal ["--log-driver", "json-file", "--log-opt", "max-size=\"10m\""], @config.logging_args + end + test "erb evaluation of yml config" do config = Mrsk::Configuration.create_from Pathname.new(File.expand_path("fixtures/deploy.erb.yml", __dir__)) assert_equal "my-user", config.registry["username"] @@ -181,6 +186,6 @@ class ConfigurationTest < ActiveSupport::TestCase end test "to_h" do - assert_equal({ :roles=>["web"], :hosts=>["1.1.1.1", "1.1.1.2"], :primary_host=>"1.1.1.1", :version=>"missing", :repository=>"dhh/app", :absolute_image=>"dhh/app:missing", :service_with_version=>"app-missing", :env_args=>["-e", "REDIS_URL=\"redis://x/y\""], :ssh_options=>{:user=>"root", :auth_methods=>["publickey"]}, :volume_args=>["--volume", "/local/path:/container/path"], :healthcheck=>{"path"=>"/up", "port"=>3000 }}, @config.to_h) + assert_equal({ :roles=>["web"], :hosts=>["1.1.1.1", "1.1.1.2"], :primary_host=>"1.1.1.1", :version=>"missing", :repository=>"dhh/app", :absolute_image=>"dhh/app:missing", :service_with_version=>"app-missing", :env_args=>["-e", "REDIS_URL=\"redis://x/y\""], :ssh_options=>{:user=>"root", :auth_methods=>["publickey"]}, :volume_args=>["--volume", "/local/path:/container/path"], :logging=>{"driver"=>"json-file", "options"=>{"max-size"=>"10m"}}, :healthcheck=>{"path"=>"/up", "port"=>3000 }}, @config.to_h) end end From 662873de49b3ad994efb1a016126e2fea9a0e479 Mon Sep 17 00:00:00 2001 From: Samuel Sieg Date: Sun, 19 Mar 2023 09:48:54 +0100 Subject: [PATCH 11/32] Add logging to README --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 56d26116..c73cbd34 100644 --- a/README.md +++ b/README.md @@ -321,6 +321,20 @@ servers: That'll start the job containers with `docker run ... --cap-add --cpu-count 4 ...`. +### Configuring logging + +You can configure the logging driver and options passed to Docker using `logging`: + +```yaml +logging: + driver: awslogs + options: + awslogs-region: "eu-central-2" + awslogs-group: "koleo-app" +``` + +If nothing is configured, Docker will use its default driver `json-file`. + ### Using remote builder for native multi-arch If you're developing on ARM64 (like Apple Silicon), but you want to deploy on AMD64 (x86 64-bit), you can use multi-architecture images. By default, MRSK will setup a local buildx configuration that does this through QEMU emulation. But this can be quite slow, especially on the first build. From b635b3198f565750cb0872b6745dee082885d7a7 Mon Sep 17 00:00:00 2001 From: Samuel Sieg Date: Sun, 19 Mar 2023 09:49:23 +0100 Subject: [PATCH 12/32] Fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c73cbd34..9c3f92b8 100644 --- a/README.md +++ b/README.md @@ -330,7 +330,7 @@ logging: driver: awslogs options: awslogs-region: "eu-central-2" - awslogs-group: "koleo-app" + awslogs-group: "my-app" ``` If nothing is configured, Docker will use its default driver `json-file`. From 43d1ecc94bfddf7cf83f10ebb3694e52304dab58 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Mon, 20 Mar 2023 17:33:13 +0100 Subject: [PATCH 13/32] Fix other tests --- test/cli/app_test.rb | 40 +++++++++++++--------------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index badc3a0d..8636de14 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -63,7 +63,19 @@ class CliAppTest < CliTestCase test "remove_container" do run_command("remove_container", "1234567").tap do |output| - assert_match /#{Regexp.escape("docker container ls --all --filter name=app-web-1234567 --quiet | xargs docker container rm")}/, output + assert_match "docker container ls --all --filter name=app-web-1234567 --quiet | xargs docker container rm", output + end + end + + test "remove_containers" do + run_command("remove_containers").tap do |output| + assert_match "docker container prune --force --filter label=service=app", output + end + end + + test "remove_images" do + run_command("remove_images").tap do |output| + assert_match "docker image prune --all --force --filter label=service=app", output end end @@ -106,32 +118,6 @@ class CliAppTest < CliTestCase assert_match "docker ps --quiet --filter label=service=app | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow") end - test "remove" do - Mrsk::Cli::App.any_instance.expects(:stop) - Mrsk::Cli::App.any_instance.expects(:remove_containers) - Mrsk::Cli::App.any_instance.expects(:remove_images) - - run_command("remove") - end - - test "remove_container" do - run_command("remove_container", "1234567").tap do |output| - assert_match "docker container ls --all --filter name=app-1234567 --quiet | xargs docker container rm", output - end - end - - test "remove_containers" do - run_command("remove_containers").tap do |output| - assert_match "docker container prune --force --filter label=service=app", output - end - end - - test "remove_images" do - run_command("remove_images").tap do |output| - assert_match "docker image prune --all --force --filter label=service=app", output - end - end - test "version" do run_command("version").tap do |output| assert_match "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1", output From 60faf27a05f072d333279c0ce4db7c586ab07e68 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Mon, 20 Mar 2023 17:40:36 +0100 Subject: [PATCH 14/32] More resilient tests --- test/cli/prune_test.rb | 6 ++---- test/cli/registry_test.rb | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/test/cli/prune_test.rb b/test/cli/prune_test.rb index 15e54575..4ff9eedc 100644 --- a/test/cli/prune_test.rb +++ b/test/cli/prune_test.rb @@ -10,15 +10,13 @@ class CliPruneTest < CliTestCase test "images" do run_command("images").tap do |output| - assert_match "docker image prune --all --force --filter label=service=app --filter until=168h on 1.1.1.1", output - assert_match "docker image prune --all --force --filter label=service=app --filter until=168h on 1.1.1.2", output + assert_match /docker image prune --all --force --filter label=service=app --filter until=168h on 1.1.1.\d/, output end end test "containers" do run_command("containers").tap do |output| - assert_match "docker container prune --force --filter label=service=app --filter until=72h on 1.1.1.1", output - assert_match "docker container prune --force --filter label=service=app --filter until=72h on 1.1.1.2", output + assert_match /docker container prune --force --filter label=service=app --filter until=72h on 1.1.1.\d/, output end end diff --git a/test/cli/registry_test.rb b/test/cli/registry_test.rb index d100402c..0647f873 100644 --- a/test/cli/registry_test.rb +++ b/test/cli/registry_test.rb @@ -4,15 +4,13 @@ class CliRegistryTest < CliTestCase test "login" do run_command("login").tap do |output| assert_match /docker login -u \[REDACTED\] -p \[REDACTED\] as .*@localhost/, output - assert_match "docker login -u [REDACTED] -p [REDACTED] on 1.1.1.1", output - assert_match "docker login -u [REDACTED] -p [REDACTED] on 1.1.1.2", output + assert_match /docker login -u \[REDACTED\] -p \[REDACTED\] on 1.1.1.\d/, output end end test "logout" do run_command("logout").tap do |output| - assert_match "docker logout on 1.1.1.1", output - assert_match "docker logout on 1.1.1.2", output + assert_match /docker logout on 1.1.1.\d/, output end end From e7e3cd98eb76f10db8f51032c2430a4ef0308347 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Thu, 23 Mar 2023 15:16:10 +0100 Subject: [PATCH 15/32] Fix tests --- test/commands/app_test.rb | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index 8511b273..a20ffc6f 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -13,7 +13,7 @@ class CommandsAppTest < ActiveSupport::TestCase test "run" do assert_equal \ - "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -37,7 +37,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:servers] = { "web" => [ "1.1.1.1" ], "jobs" => { "hosts" => [ "1.1.1.2" ], "cmd" => "bin/jobs", "options" => { "mount" => "somewhere", "cap-add" => true } } } assert_equal \ "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-jobs-999 -e MRSK_CONTAINER_NAME=\"app-jobs-999\" -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs", - new_command.run(role: :jobs).join(" ") + new_command(role: "jobs").run.join(" ") end test "start" do @@ -49,7 +49,7 @@ class CommandsAppTest < ActiveSupport::TestCase test "start with destination" do @destination = "staging" assert_equal \ - "docker start app-staging-999", + "docker start app-web-staging-999", new_command.start.join(" ") end @@ -74,7 +74,7 @@ class CommandsAppTest < ActiveSupport::TestCase test "info with destination" do @destination = "staging" assert_equal \ - "docker ps --filter label=service=app --filter label=role=web --filter label=destination=staging", + "docker ps --filter label=service=app --filter label=destination=staging --filter label=role=web", new_command.info.join(" ") end @@ -165,20 +165,20 @@ class CommandsAppTest < ActiveSupport::TestCase test "current_container_id" do assert_equal \ - "docker ps --quiet --filter label=service=app--filter label=role=web", + "docker ps --quiet --filter label=service=app --filter label=role=web", new_command.current_container_id.join(" ") end test "current_container_id with destination" do @destination = "staging" assert_equal \ - "docker ps --quiet --filter label=service=app--filter label=role=web --filter label=destination=staging", + "docker ps --quiet --filter label=service=app --filter label=destination=staging --filter label=role=web", new_command.current_container_id.join(" ") end test "container_id_for" do assert_equal \ - "docker container ls --all --filter name=app-web-999 --quiet", + "docker container ls --all --filter name=app-999 --quiet", new_command.container_id_for(container_name: "app-999").join(" ") end @@ -196,46 +196,46 @@ class CommandsAppTest < ActiveSupport::TestCase test "list_containers" do assert_equal \ - "docker container ls --all --filter label=service=app", + "docker container ls --all --filter label=service=app --filter label=role=web", new_command.list_containers.join(" ") end test "list_containers with destination" do @destination = "staging" assert_equal \ - "docker container ls --all --filter label=service=app --filter label=destination=staging", + "docker container ls --all --filter label=service=app --filter label=destination=staging --filter label=role=web", new_command.list_containers.join(" ") end test "list_container_names" do assert_equal \ - "docker container ls --all --filter label=service=app --format '{{ .Names }}'", + "docker container ls --all --filter label=service=app --filter label=role=web --format '{{ .Names }}'", new_command.list_container_names.join(" ") end test "remove_container" do assert_equal \ - "docker container ls --all --filter name=app-999 --quiet | xargs docker container rm", + "docker container ls --all --filter name=app-web-999 --quiet | xargs docker container rm", new_command.remove_container(version: "999").join(" ") end test "remove_container with destination" do @destination = "staging" assert_equal \ - "docker container ls --all --filter name=app-staging-999 --quiet | xargs docker container rm", + "docker container ls --all --filter name=app-web-staging-999 --quiet | xargs docker container rm", new_command.remove_container(version: "999").join(" ") end test "remove_containers" do assert_equal \ - "docker container prune --force --filter label=service=app", + "docker container prune --force --filter label=service=app --filter label=role=web", new_command.remove_containers.join(" ") end test "remove_containers with destination" do @destination = "staging" assert_equal \ - "docker container prune --force --filter label=service=app --filter label=destination=staging", + "docker container prune --force --filter label=service=app --filter label=destination=staging --filter label=role=web", new_command.remove_containers.join(" ") end @@ -247,19 +247,19 @@ class CommandsAppTest < ActiveSupport::TestCase test "remove_images" do assert_equal \ - "docker image prune --all --force --filter label=service=app", + "docker image prune --all --force --filter label=service=app --filter label=role=web", new_command.remove_images.join(" ") end test "remove_images with destination" do @destination = "staging" assert_equal \ - "docker image prune --all --force --filter label=service=app --filter label=destination=staging", + "docker image prune --all --force --filter label=service=app --filter label=destination=staging --filter label=role=web", new_command.remove_images.join(" ") end private - def new_command - Mrsk::Commands::App.new(Mrsk::Configuration.new(@config, destination: @destination, role: "web", version: "999")) + def new_command(role: "web") + Mrsk::Commands::App.new(Mrsk::Configuration.new(@config, destination: @destination, version: "999"), role: role) end end From 7e1596e72225672fd198354fbacc9059bc1c5a24 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Thu, 23 Mar 2023 15:36:02 +0100 Subject: [PATCH 16/32] Fix flaky test --- test/cli/main_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index f346dd02..5e761e1b 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -88,7 +88,7 @@ class CliMainTest < CliTestCase test "rollback good version" do Mrsk::Cli::Main.any_instance.stubs(:container_name_available?).returns(true) - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1").returns("version-to-rollback\n").times(2) + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1").returns("version-to-rollback\n").at_least_once run_command("rollback", "123").tap do |output| assert_match "Start version 123", output @@ -99,7 +99,7 @@ class CliMainTest < CliTestCase test "rollback without old version" do Mrsk::Cli::Main.any_instance.stubs(:container_name_available?).returns(true) - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1").returns("").times(2) + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info).with(:docker, :ps, "--filter", "label=service=app", "--format", "\"{{.Names}}\"", "|", "sed 's/-/\\n/g'", "|", "tail -n 1").returns("").at_least_once run_command("rollback", "123").tap do |output| assert_match "Start version 123", output From 5d5d58a4ec0bd625cc5908b5e8690a9156a00ebb Mon Sep 17 00:00:00 2001 From: Javier Aranda Date: Thu, 23 Mar 2023 23:56:59 +0100 Subject: [PATCH 17/32] #142 Allow to customize container options in accessories --- README.md | 3 +++ lib/mrsk/commands/accessory.rb | 4 +++- lib/mrsk/configuration/accessory.rb | 10 +++++++++- test/configuration/accessory_test.rb | 10 +++++++++- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 379dcbee..0bef1612 100644 --- a/README.md +++ b/README.md @@ -500,6 +500,9 @@ accessories: - MYSQL_ROOT_PASSWORD volumes: - /var/lib/mysql:/var/lib/mysql + options: + cpus: 4 + memory: "2GB" redis: image: redis:latest host: 1.1.1.4 diff --git a/lib/mrsk/commands/accessory.rb b/lib/mrsk/commands/accessory.rb index 76d4fa96..04852d3d 100644 --- a/lib/mrsk/commands/accessory.rb +++ b/lib/mrsk/commands/accessory.rb @@ -1,6 +1,7 @@ class Mrsk::Commands::Accessory < Mrsk::Commands::Base attr_reader :accessory_config - delegate :service_name, :image, :host, :port, :files, :directories, :publish_args, :env_args, :volume_args, :label_args, to: :accessory_config + delegate :service_name, :image, :host, :port, :files, :directories, :publish_args, :env_args, :volume_args, + :label_args, :option_args, to: :accessory_config def initialize(config, name:) super(config) @@ -17,6 +18,7 @@ class Mrsk::Commands::Accessory < Mrsk::Commands::Base *env_args, *volume_args, *label_args, + *option_args, image end diff --git a/lib/mrsk/configuration/accessory.rb b/lib/mrsk/configuration/accessory.rb index f1051423..73c8a7dc 100644 --- a/lib/mrsk/configuration/accessory.rb +++ b/lib/mrsk/configuration/accessory.rb @@ -1,5 +1,5 @@ class Mrsk::Configuration::Accessory - delegate :argumentize, :argumentize_env_with_secrets, to: Mrsk::Utils + delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Mrsk::Utils attr_accessor :name, :specifics @@ -67,6 +67,14 @@ class Mrsk::Configuration::Accessory argumentize "--volume", volumes end + def option_args + if args = specifics["options"] + optionize args + else + [] + end + end + private attr_accessor :config diff --git a/test/configuration/accessory_test.rb b/test/configuration/accessory_test.rb index 7062de37..fd070949 100644 --- a/test/configuration/accessory_test.rb +++ b/test/configuration/accessory_test.rb @@ -39,7 +39,11 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase }, "volumes" => [ "/var/lib/redis:/data" - ] + ], + "options" => { + "cpus" => 4, + "memory" => "2GB" + } } } } @@ -104,4 +108,8 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase test "directories" do assert_equal({"$PWD/app-mysql/data"=>"/var/lib/mysql"}, @config.accessory(:mysql).directories) end + + test "options" do + assert_equal ["--cpus", "\"4\"", "--memory", "\"2GB\""], @config.accessory(:redis).option_args + end end From 7369be48ff3592b49bcf59a98fcbd7cf737aa23d Mon Sep 17 00:00:00 2001 From: Samuel Sieg Date: Fri, 24 Mar 2023 09:10:36 +0100 Subject: [PATCH 18/32] Ensure default log option `max-size=10m` --- lib/mrsk/configuration.rb | 8 +++++--- test/cli/accessory_test.rb | 6 +++--- test/cli/traefik_test.rb | 2 +- test/commands/accessory_test.rb | 8 ++++---- test/commands/app_test.rb | 12 ++++++------ test/commands/traefik_test.rb | 22 +++++++++++----------- test/configuration_test.rb | 12 ++++++++---- 7 files changed, 38 insertions(+), 32 deletions(-) diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index 789b9d1d..02bdf088 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -12,6 +12,8 @@ class Mrsk::Configuration attr_accessor :destination attr_accessor :raw_config + MAX_LOG_SIZE = "10m" + class << self def create_from(config_file:, destination: nil, version: nil) raw_config = load_config_files(config_file, *destination_config_file(config_file, destination)) @@ -123,10 +125,10 @@ class Mrsk::Configuration def logging_args if raw_config.logging.present? - [ "--log-driver", raw_config.logging["driver"] || "json-file" ] + + [ "--log-driver", raw_config.logging["driver"] ] + argumentize("--log-opt", raw_config.logging["options"]) else - [] + argumentize("--log-opt", { "max-size" => MAX_LOG_SIZE }) end end @@ -182,7 +184,7 @@ class Mrsk::Configuration ssh_options: ssh_options, builder: raw_config.builder, accessories: raw_config.accessories, - logging: raw_config.logging, + logging: logging_args, healthcheck: healthcheck }.compact end diff --git a/test/cli/accessory_test.rb b/test/cli/accessory_test.rb index 8d2fe435..28fadc7e 100644 --- a/test/cli/accessory_test.rb +++ b/test/cli/accessory_test.rb @@ -7,7 +7,7 @@ class CliAccessoryTest < CliTestCase run_command("boot", "mysql").tap do |output| assert_match /docker login.*on 1.1.1.3/, output - assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=10m --publish 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output end end @@ -20,8 +20,8 @@ class CliAccessoryTest < CliTestCase run_command("boot", "all").tap do |output| assert_match /docker login.*on 1.1.1.3/, output assert_match /docker login.*on 1.1.1.4/, output - assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=10m --publish 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output - assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=10m --publish 6379:6379 --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.4", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.4", output end end diff --git a/test/cli/traefik_test.rb b/test/cli/traefik_test.rb index e1d86dee..e8626b06 100644 --- a/test/cli/traefik_test.rb +++ b/test/cli/traefik_test.rb @@ -3,7 +3,7 @@ require_relative "cli_test_case" class CliTraefikTest < CliTestCase test "boot" do run_command("boot").tap do |output| - assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG", output + assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" traefik --providers.docker --log.level=DEBUG", output end end diff --git a/test/commands/accessory_test.rb b/test/commands/accessory_test.rb index ac38b38e..d14b5cef 100644 --- a/test/commands/accessory_test.rb +++ b/test/commands/accessory_test.rb @@ -49,15 +49,15 @@ class CommandsAccessoryTest < ActiveSupport::TestCase test "run" do assert_equal \ - "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=10m --publish 3306:3306 -e MYSQL_ROOT_PASSWORD=\"secret123\" -e MYSQL_ROOT_HOST=\"%\" --label service=\"app-mysql\" private.registry/mysql:8.0", + "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 -e MYSQL_ROOT_PASSWORD=\"secret123\" -e MYSQL_ROOT_HOST=\"%\" --label service=\"app-mysql\" private.registry/mysql:8.0", new_command(:mysql).run.join(" ") assert_equal \ - "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=10m --publish 6379:6379 -e SOMETHING=\"else\" --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest", + "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 -e SOMETHING=\"else\" --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest", new_command(:redis).run.join(" ") assert_equal \ - "docker run --name app-busybox --detach --restart unless-stopped --log-opt max-size=10m --label service=\"app-busybox\" busybox:latest", + "docker run --name app-busybox --detach --restart unless-stopped --log-opt max-size=\"10m\" --label service=\"app-busybox\" busybox:latest", new_command(:busybox).run.join(" ") end @@ -93,7 +93,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase end test "execute in new container over ssh" do - @mysql.stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do + new_command(:mysql).stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do assert_match %r|docker run -it --rm -e MYSQL_ROOT_PASSWORD=\"secret123\" -e MYSQL_ROOT_HOST=\"%\" private.registry/mysql:8.0 mysql -u root|, new_command(:mysql).execute_in_new_container_over_ssh("mysql", "-u", "root") end diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index 00fa97b5..65185636 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -13,7 +13,7 @@ class CommandsAppTest < ActiveSupport::TestCase test "run" do assert_equal \ - "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -21,7 +21,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:volumes] = ["/local/path:/container/path" ] assert_equal \ - "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -29,7 +29,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:healthcheck] = { "path" => "/healthz" } assert_equal \ - "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/healthz\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/healthz\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -37,15 +37,15 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:servers] = { "web" => [ "1.1.1.1" ], "jobs" => { "hosts" => [ "1.1.1.2" ], "cmd" => "bin/jobs", "options" => { "mount" => "somewhere", "cap-add" => true } } } assert_equal \ - "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs", + "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs", new_command.run(role: :jobs).join(" ") end test "run with logging config" do - @config[:logging] = { "driver" => "local", "options" => { "max-size" => "10m", "max-file" => "3" } } + @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } assert_equal \ - "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --log-driver local --log-opt max-size=\"10m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --log-driver local --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", new_command.run.join(" ") end diff --git a/test/commands/traefik_test.rb b/test/commands/traefik_test.rb index 523a1de0..a0d1b8ae 100644 --- a/test/commands/traefik_test.rb +++ b/test/commands/traefik_test.rb @@ -10,45 +10,45 @@ class CommandsTraefikTest < ActiveSupport::TestCase test "run" do assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", + "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") @config[:traefik]["host_port"] = "8080" assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", + "docker run --name traefik --detach --restart unless-stopped --publish 8080:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") end test "run with ports configured" do assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --log-opt max-size=10m --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", + "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") @config[:traefik]["options"] = {"publish" => %w[9000:9000 9001:9001]} assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --log-opt max-size=10m --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --publish \"9000:9000\" --publish \"9001:9001\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", + "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --publish \"9000:9000\" --publish \"9001:9001\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") end test "run with volumes configured" do assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --log-opt max-size=10m --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", + "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json] } assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --log-opt max-size=10m --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", + "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") end test "run with several options configured" do assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --log-opt max-size=10m --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", + "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") @config[:traefik]["options"] = {"volume" => %w[./letsencrypt/acme.json:/letsencrypt/acme.json], "publish" => %w[8080:8080], "memory" => "512m"} assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --log-opt max-size=10m --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", + "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" --volume \"./letsencrypt/acme.json:/letsencrypt/acme.json\" --publish \"8080:8080\" --memory \"512m\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") end @@ -56,15 +56,15 @@ class CommandsTraefikTest < ActiveSupport::TestCase @config.delete(:traefik) assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock traefik --providers.docker --log.level=DEBUG", + "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-opt max-size=\"10m\" traefik --providers.docker --log.level=DEBUG", new_command.run.join(" ") end test "run with logging config" do - @config[:logging] = { "driver" => "local", "options" => { "max-size" => "10m", "max-file" => "3" } } + @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-driver local --log-opt max-size=\"10m\" --log-opt max-file=\"3\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", + "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-driver local --log-opt max-size=\"100m\" --log-opt max-file=\"3\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") end diff --git a/test/configuration_test.rb b/test/configuration_test.rb index 53558c93..a47b5cee 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -11,7 +11,6 @@ class ConfigurationTest < ActiveSupport::TestCase env: { "REDIS_URL" => "redis://x/y" }, servers: [ "1.1.1.1", "1.1.1.2" ], volumes: ["/local/path:/container/path"], - logging: { "driver" => "json-file", "options" => { "max-size" => "10m" } } } @config = Mrsk::Configuration.new(@deploy) @@ -210,8 +209,13 @@ class ConfigurationTest < ActiveSupport::TestCase assert_equal ["--volume", "/local/path:/container/path"], @config.volume_args end - test "logging_args" do - assert_equal ["--log-driver", "json-file", "--log-opt", "max-size=\"10m\""], @config.logging_args + test "logging args default" do + assert_equal ["--log-opt", "max-size=\"10m\""], @config.logging_args + end + + test "logging args with custom config" do + config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(logging: { "driver" => "local", "options" => { "max-file" => 5 } }) }) + assert_equal ["--log-driver", "local", "--log-opt", "max-file=\"5\""], @config.logging_args end test "erb evaluation of yml config" do @@ -238,6 +242,6 @@ class ConfigurationTest < ActiveSupport::TestCase end test "to_h" do - assert_equal({ :roles=>["web"], :hosts=>["1.1.1.1", "1.1.1.2"], :primary_host=>"1.1.1.1", :version=>"missing", :repository=>"dhh/app", :absolute_image=>"dhh/app:missing", :service_with_version=>"app-missing", :env_args=>["-e", "REDIS_URL=\"redis://x/y\""], :ssh_options=>{:user=>"root", :auth_methods=>["publickey"]}, :volume_args=>["--volume", "/local/path:/container/path"], :logging=>{"driver"=>"json-file", "options"=>{"max-size"=>"10m"}}, :healthcheck=>{"path"=>"/up", "port"=>3000 }}, @config.to_h) + assert_equal({ :roles=>["web"], :hosts=>["1.1.1.1", "1.1.1.2"], :primary_host=>"1.1.1.1", :version=>"missing", :repository=>"dhh/app", :absolute_image=>"dhh/app:missing", :service_with_version=>"app-missing", :env_args=>["-e", "REDIS_URL=\"redis://x/y\""], :ssh_options=>{:user=>"root", :auth_methods=>["publickey"]}, :volume_args=>["--volume", "/local/path:/container/path"], :logging=>["--log-opt", "max-size=\"10m\""], :healthcheck=>{"path"=>"/up", "port"=>3000 }}, @config.to_h) end end From 3392fc6c1becd89bd4d0913251211635885b17da Mon Sep 17 00:00:00 2001 From: Samuel Sieg Date: Fri, 24 Mar 2023 09:15:03 +0100 Subject: [PATCH 19/32] Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b67e4218..d47c3303 100644 --- a/README.md +++ b/README.md @@ -333,7 +333,7 @@ logging: awslogs-group: "my-app" ``` -If nothing is configured, Docker will use its default driver `json-file`. +If nothing is configured, the default option `max-size=10m` is used for all containers. The default logging driver of Docker is `json-file`. ### Using remote builder for native multi-arch From ba5bdf95eced2c05acdd023d3796d7dec2424aa3 Mon Sep 17 00:00:00 2001 From: Samuel Sieg Date: Fri, 24 Mar 2023 09:15:30 +0100 Subject: [PATCH 20/32] Improve test --- test/configuration_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/configuration_test.rb b/test/configuration_test.rb index a47b5cee..7b22ee2a 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -214,8 +214,8 @@ class ConfigurationTest < ActiveSupport::TestCase end test "logging args with custom config" do - config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(logging: { "driver" => "local", "options" => { "max-file" => 5 } }) }) - assert_equal ["--log-driver", "local", "--log-opt", "max-file=\"5\""], @config.logging_args + config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(logging: { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => 5 } }) }) + assert_equal ["--log-driver", "local", "--log-opt", "max-size=\"100m\"", "--log-opt", "max-file=\"5\""], @config.logging_args end test "erb evaluation of yml config" do From 20a6bc31cd91e1a1cb7183bd65ca316925dce6d3 Mon Sep 17 00:00:00 2001 From: Samuel Sieg Date: Fri, 24 Mar 2023 09:15:37 +0100 Subject: [PATCH 21/32] Undo change --- test/configuration_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/configuration_test.rb b/test/configuration_test.rb index 7b22ee2a..75b9de14 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -10,7 +10,7 @@ class ConfigurationTest < ActiveSupport::TestCase registry: { "username" => "dhh", "password" => "secret" }, env: { "REDIS_URL" => "redis://x/y" }, servers: [ "1.1.1.1", "1.1.1.2" ], - volumes: ["/local/path:/container/path"], + volumes: ["/local/path:/container/path"] } @config = Mrsk::Configuration.new(@deploy) From c3de89bb5916b0e7a7b5127d448cad7bbe4ca867 Mon Sep 17 00:00:00 2001 From: Samuel Sieg Date: Fri, 24 Mar 2023 09:19:13 +0100 Subject: [PATCH 22/32] Add accessory test --- test/commands/accessory_test.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/commands/accessory_test.rb b/test/commands/accessory_test.rb index d14b5cef..84590d56 100644 --- a/test/commands/accessory_test.rb +++ b/test/commands/accessory_test.rb @@ -61,6 +61,14 @@ class CommandsAccessoryTest < ActiveSupport::TestCase new_command(:busybox).run.join(" ") end + test "run with logging config" do + @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } + + assert_equal \ + "docker run --name app-busybox --detach --restart unless-stopped --log-driver local --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app-busybox\" busybox:latest", + new_command(:busybox).run.join(" ") + end + test "start" do assert_equal \ "docker container start app-mysql", From 9c27ead21fd936f5d0005efbb3297d419c8d9058 Mon Sep 17 00:00:00 2001 From: Samuel Sieg Date: Fri, 24 Mar 2023 09:38:02 +0100 Subject: [PATCH 23/32] Ensure it also works when configuring just log options without setting a driver --- lib/mrsk/configuration.rb | 4 ++-- test/commands/accessory_test.rb | 2 +- test/commands/app_test.rb | 2 +- test/commands/traefik_test.rb | 2 +- test/configuration_test.rb | 9 +++++++-- 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index 02bdf088..cf4f672c 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -7,7 +7,7 @@ require "net/ssh/proxy/jump" class Mrsk::Configuration delegate :service, :image, :servers, :env, :labels, :registry, :builder, :logging, to: :raw_config, allow_nil: true - delegate :argumentize, :argumentize_env_with_secrets, to: Mrsk::Utils + delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Mrsk::Utils attr_accessor :destination attr_accessor :raw_config @@ -125,7 +125,7 @@ class Mrsk::Configuration def logging_args if raw_config.logging.present? - [ "--log-driver", raw_config.logging["driver"] ] + + optionize({ "log-driver" => raw_config.logging["driver"] }.compact) + argumentize("--log-opt", raw_config.logging["options"]) else argumentize("--log-opt", { "max-size" => MAX_LOG_SIZE }) diff --git a/test/commands/accessory_test.rb b/test/commands/accessory_test.rb index 84590d56..71a3058b 100644 --- a/test/commands/accessory_test.rb +++ b/test/commands/accessory_test.rb @@ -65,7 +65,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } assert_equal \ - "docker run --name app-busybox --detach --restart unless-stopped --log-driver local --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app-busybox\" busybox:latest", + "docker run --name app-busybox --detach --restart unless-stopped --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app-busybox\" busybox:latest", new_command(:busybox).run.join(" ") end diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index 65185636..2904e7dc 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -45,7 +45,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } assert_equal \ - "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --log-driver local --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", new_command.run.join(" ") end diff --git a/test/commands/traefik_test.rb b/test/commands/traefik_test.rb index a0d1b8ae..3f6084d1 100644 --- a/test/commands/traefik_test.rb +++ b/test/commands/traefik_test.rb @@ -64,7 +64,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-driver local --log-opt max-size=\"100m\" --log-opt max-file=\"3\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", + "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" traefik --providers.docker --log.level=DEBUG --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") end diff --git a/test/configuration_test.rb b/test/configuration_test.rb index 75b9de14..74739bc7 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -213,9 +213,14 @@ class ConfigurationTest < ActiveSupport::TestCase assert_equal ["--log-opt", "max-size=\"10m\""], @config.logging_args end - test "logging args with custom config" do + test "logging args with configured options" do + config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(logging: { "options" => { "max-size" => "100m", "max-file" => 5 } }) }) + assert_equal ["--log-opt", "max-size=\"100m\"", "--log-opt", "max-file=\"5\""], @config.logging_args + end + + test "logging args with configured driver and options" do config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(logging: { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => 5 } }) }) - assert_equal ["--log-driver", "local", "--log-opt", "max-size=\"100m\"", "--log-opt", "max-file=\"5\""], @config.logging_args + assert_equal ["--log-driver", "\"local\"", "--log-opt", "max-size=\"100m\"", "--log-opt", "max-file=\"5\""], @config.logging_args end test "erb evaluation of yml config" do From 494e29d67298fe8718f2e14076c90760833a2ec9 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Fri, 24 Mar 2023 14:35:17 +0100 Subject: [PATCH 24/32] Fix tests --- test/cli/app_test.rb | 9 +++------ test/commander_test.rb | 17 +++-------------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index fd6016f2..fa132007 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -3,10 +3,7 @@ require_relative "cli_test_case" class CliAppTest < CliTestCase test "boot" do # Stub current version fetch - SSHKit::Backend::Abstract.any_instance.stubs(:capture) - .returns("999") # new version - .then - .returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture).returns("123") # old version run_command("boot").tap do |output| assert_match "docker run --detach --restart unless-stopped", output @@ -27,7 +24,7 @@ class CliAppTest < CliTestCase run_command("boot").tap do |output| assert_match "Rebooting container with same version latest already deployed", output # Can't start what's already running - assert_match "docker container ls --all --filter name=app-latest --quiet | xargs docker container rm", output # Remove old container + assert_match "docker container ls --all --filter name=app-web-latest --quiet | xargs docker container rm", output # Remove old container assert_match "docker run", output # Start new container end ensure @@ -80,7 +77,7 @@ class CliAppTest < CliTestCase test "exec" do run_command("exec", "ruby -v").tap do |output| - assert_match "docker run --rm dhh/app:999 ruby -v", output + assert_match "docker run --rm dhh/app:latest ruby -v", output end end diff --git a/test/commander_test.rb b/test/commander_test.rb index c7924f9b..245350c2 100644 --- a/test/commander_test.rb +++ b/test/commander_test.rb @@ -11,16 +11,6 @@ class CommanderTest < ActiveSupport::TestCase assert_equal Mrsk::Configuration, @mrsk.config.class end - test "commit hash as version" do - assert_equal `git rev-parse HEAD`.strip, @mrsk.config.version - end - - test "commit hash as version but not in git" do - @mrsk.expects(:system).with("git rev-parse").returns(nil) - error = assert_raises(RuntimeError) { @mrsk.config } - assert_match /no git repository found/, error.message - end - test "overwriting hosts" do assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @mrsk.hosts @@ -29,7 +19,7 @@ class CommanderTest < ActiveSupport::TestCase end test "filtering hosts by filtering roles" do - assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], @mrsk.hosts + assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @mrsk.hosts @mrsk.specific_roles = [ "web" ] assert_equal [ "1.1.1.1", "1.1.1.2" ], @mrsk.hosts @@ -50,7 +40,7 @@ class CommanderTest < ActiveSupport::TestCase end test "overwriting hosts with primary" do - assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3" ], @mrsk.hosts + assert_equal [ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ], @mrsk.hosts @mrsk.specific_primary! assert_equal [ "1.1.1.1" ], @mrsk.hosts @@ -62,8 +52,7 @@ class CommanderTest < ActiveSupport::TestCase end test "roles_on" do - assert_equal [ "web", "workers" ], @mrsk.roles_on("1.1.1.1") - assert_equal [ "web" ], @mrsk.roles_on("1.1.1.2") + assert_equal [ "web" ], @mrsk.roles_on("1.1.1.1") assert_equal [ "workers" ], @mrsk.roles_on("1.1.1.3") end end From 9b43a6b23b445606408efbf32ac059d0aaf4ba1d Mon Sep 17 00:00:00 2001 From: Jacopo Date: Fri, 24 Mar 2023 14:57:49 +0100 Subject: [PATCH 25/32] Customizable stop wait time Configurable via a global `stop_wait_time` option. The default is `10` which matches Docker defaults. --- README.md | 9 +++++++++ lib/mrsk/commands/app.rb | 4 +++- lib/mrsk/configuration.rb | 3 ++- test/commands/app_test.rb | 4 ++-- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1ffd09f1..24fd2328 100644 --- a/README.md +++ b/README.md @@ -333,6 +333,15 @@ servers: That'll start the job containers with `docker run ... --cap-add --cpu-count 4 ...`. +### Using a different stop wait time + +On a new deploy, each old running container is gracefully shut down with a `SIGTERM`, and after a grace period of `10` seconds a `SIGKILL` is sent. +You can configure this value via the `stop_wait_time` option: + +```yaml +stop_wait_time: 30 +``` + ### Using remote builder for native multi-arch If you're developing on ARM64 (like Apple Silicon), but you want to deploy on AMD64 (x86 64-bit), you can use multi-architecture images. By default, MRSK will setup a local buildx configuration that does this through QEMU emulation. But this can be quite slow, especially on the first build. diff --git a/lib/mrsk/commands/app.rb b/lib/mrsk/commands/app.rb index 9f9e987c..5d4d36d1 100644 --- a/lib/mrsk/commands/app.rb +++ b/lib/mrsk/commands/app.rb @@ -27,10 +27,12 @@ class Mrsk::Commands::App < Mrsk::Commands::Base docker :start, service_with_version_and_destination_and_role end + DEFAULT_STOP_WAIT_TIME = 10 + def stop(version: nil) pipe \ version ? container_id_for_version(version) : current_container_id, - xargs(docker(:stop)) + xargs(docker(:stop, "-t", config.stop_wait_time || DEFAULT_STOP_WAIT_TIME)) end def info diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index 71f56b37..a9f72afe 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -6,7 +6,8 @@ require "erb" require "net/ssh/proxy/jump" class Mrsk::Configuration - delegate :service, :image, :servers, :env, :labels, :registry, :builder, to: :raw_config, allow_nil: true + delegate :service, :image, :servers, :env, :labels, :registry, :builder, :stop_wait_time, + to: :raw_config, allow_nil: true delegate :argumentize, :argumentize_env_with_secrets, to: Mrsk::Utils attr_accessor :destination diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index 2a18b9c6..26c6fa04 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -55,13 +55,13 @@ class CommandsAppTest < ActiveSupport::TestCase test "stop" do assert_equal \ - "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop", + "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop -t 10", new_command.stop.join(" ") end test "stop with version" do assert_equal \ - "docker container ls --all --filter name=app-web-123 --quiet | xargs docker stop", + "docker container ls --all --filter name=app-web-123 --quiet | xargs docker stop -t 10", new_command.stop(version: "123").join(" ") end From 4ab5199853e0021af3ba2bc6dbf7f1de4d847e46 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Fri, 24 Mar 2023 15:16:15 +0100 Subject: [PATCH 26/32] Style and ordering --- lib/mrsk/commands/auditor.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/mrsk/commands/auditor.rb b/lib/mrsk/commands/auditor.rb index 9d6cf414..6915a521 100644 --- a/lib/mrsk/commands/auditor.rb +++ b/lib/mrsk/commands/auditor.rb @@ -32,26 +32,26 @@ class Mrsk::Commands::Auditor < Mrsk::Commands::Base end def tagged_record_line(line) - quote [recorded_at_tag, performer_tag, role_tag, line].compact.join(" ") + tagged_line recorded_at_tag, performer_tag, role_tag, line end def tagged_broadcast_line(line) - quote [performer_tag, role_tag, line].compact.join(" ") + tagged_line performer_tag, role_tag, line end - def role_tag - "[#{role}]" if role - end - - def performer_tag - "[#{`whoami`.strip}]" + def tagged_line(*tags_and_line) + "'#{tags_and_line.compact.join(" ")}'" end def recorded_at_tag "[#{Time.now.to_fs(:db)}]" end - def quote(tagged_line) - "'#{tagged_line}'" + def performer_tag + "[#{`whoami`.strip}]" + end + + def role_tag + "[#{role}]" if role end end From 53095a053e15728712a3042bd1fe63238ccec561 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Fri, 24 Mar 2023 14:54:41 +0100 Subject: [PATCH 27/32] Describe purpose rather than elements --- lib/mrsk/commands/app.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/mrsk/commands/app.rb b/lib/mrsk/commands/app.rb index 9f9e987c..518a9aff 100644 --- a/lib/mrsk/commands/app.rb +++ b/lib/mrsk/commands/app.rb @@ -13,8 +13,8 @@ class Mrsk::Commands::App < Mrsk::Commands::Base "--detach", "--restart unless-stopped", "--log-opt", "max-size=#{MAX_LOG_SIZE}", - "--name", service_with_version_and_destination_and_role, - "-e", "MRSK_CONTAINER_NAME=\"#{service_with_version_and_destination_and_role}\"", + "--name", container_name, + "-e", "MRSK_CONTAINER_NAME=\"#{container_name}\"", *role.env_args, *config.volume_args, *role.label_args, @@ -24,7 +24,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base end def start - docker :start, service_with_version_and_destination_and_role + docker :start, container_name end def stop(version: nil) @@ -59,7 +59,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base def execute_in_existing_container(*command, interactive: false) docker :exec, ("-it" if interactive), - service_with_version_and_destination_and_role, + container_name, *command end @@ -104,7 +104,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base def remove_container(version:) pipe \ - container_id_for(container_name: service_with_version_and_destination_and_role(version)), + container_id_for(container_name: container_name(version)), xargs(docker(:container, :rm)) end @@ -122,12 +122,12 @@ class Mrsk::Commands::App < Mrsk::Commands::Base private - def service_with_version_and_destination_and_role(version = nil) + def container_name(version = nil) [ config.service, role, config.destination, version || config.version ].compact.join("-") end def container_id_for_version(version) - container_id_for(container_name: service_with_version_and_destination_and_role(version)) + container_id_for(container_name: container_name(version)) end def filter_args From a9bb8d7376edd128b94962cb4ccf085ddcfd651b Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Fri, 24 Mar 2023 15:18:18 +0100 Subject: [PATCH 28/32] No need to replicate Docker default --- lib/mrsk/commands/app.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/mrsk/commands/app.rb b/lib/mrsk/commands/app.rb index 5d4d36d1..5fc8dda7 100644 --- a/lib/mrsk/commands/app.rb +++ b/lib/mrsk/commands/app.rb @@ -27,12 +27,10 @@ class Mrsk::Commands::App < Mrsk::Commands::Base docker :start, service_with_version_and_destination_and_role end - DEFAULT_STOP_WAIT_TIME = 10 - def stop(version: nil) pipe \ version ? container_id_for_version(version) : current_container_id, - xargs(docker(:stop, "-t", config.stop_wait_time || DEFAULT_STOP_WAIT_TIME)) + xargs(config.stop_wait_time ? docker(:stop, "-t", config.stop_wait_time) : docker(:stop)) end def info From fdb2502216f8b83ac2609b69ea99e5e71ab525ca Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Fri, 24 Mar 2023 15:22:34 +0100 Subject: [PATCH 29/32] test stop with custom stop wait time --- test/commands/app_test.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index 26c6fa04..0c277e2b 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -55,13 +55,20 @@ class CommandsAppTest < ActiveSupport::TestCase test "stop" do assert_equal \ - "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop -t 10", + "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop", + new_command.stop.join(" ") + end + + test "stop with custom stop wait time" do + @config["stop_wait_time"] = 30 + assert_equal \ + "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop -t 30", new_command.stop.join(" ") end test "stop with version" do assert_equal \ - "docker container ls --all --filter name=app-web-123 --quiet | xargs docker stop -t 10", + "docker container ls --all --filter name=app-web-123 --quiet | xargs docker stop", new_command.stop(version: "123").join(" ") end From 4044abdde164c17f2ca58591e7e1eb6b03e72206 Mon Sep 17 00:00:00 2001 From: Samuel Sieg Date: Fri, 24 Mar 2023 15:25:29 +0100 Subject: [PATCH 30/32] Fix tests --- test/commands/app_test.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index e9caa7c1..0080c556 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -13,7 +13,7 @@ class CommandsAppTest < ActiveSupport::TestCase test "run" do assert_equal \ - "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -21,7 +21,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:volumes] = ["/local/path:/container/path" ] assert_equal \ - "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -29,14 +29,14 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:healthcheck] = { "path" => "/healthz" } assert_equal \ - "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/healthz\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/healthz\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", new_command.run.join(" ") end test "run with custom options" do @config[:servers] = { "web" => [ "1.1.1.1" ], "jobs" => { "hosts" => [ "1.1.1.2" ], "cmd" => "bin/jobs", "options" => { "mount" => "somewhere", "cap-add" => true } } } assert_equal \ - "docker run --detach --restart unless-stopped --log-opt max-size=10m --name app-jobs-999 -e MRSK_CONTAINER_NAME=\"app-jobs-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs", + "docker run --detach --restart unless-stopped --name app-jobs-999 -e MRSK_CONTAINER_NAME=\"app-jobs-999\" -e RAILS_MASTER_KEY=\"456\" --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs", new_command(role: "jobs").run.join(" ") end @@ -44,7 +44,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } assert_equal \ - "docker run --detach --restart unless-stopped --name app-999 -e MRSK_CONTAINER_NAME=\"app-999\" -e RAILS_MASTER_KEY=\"456\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e MRSK_CONTAINER_NAME=\"app-web-999\" -e RAILS_MASTER_KEY=\"456\" --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.routers.app.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.services.app.loadbalancer.healthcheck.path=\"/up\" --label traefik.http.services.app.loadbalancer.healthcheck.interval=\"1s\" --label traefik.http.middlewares.app-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app.middlewares=\"app-retry@docker\" dhh/app:999", new_command.run.join(" ") end From 4fa71834ad68bacf0c6e2df03f26c95071dc0ce8 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Fri, 24 Mar 2023 15:27:11 +0100 Subject: [PATCH 31/32] Symbols! --- test/commands/app_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index 0c277e2b..53e2d238 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -60,7 +60,7 @@ class CommandsAppTest < ActiveSupport::TestCase end test "stop with custom stop wait time" do - @config["stop_wait_time"] = 30 + @config[:stop_wait_time] = 30 assert_equal \ "docker ps --quiet --filter label=service=app --filter label=role=web | xargs docker stop -t 30", new_command.stop.join(" ") From 6a27a46e5f4c588947039be66f73f54fb3d30d68 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Fri, 24 Mar 2023 15:34:34 +0100 Subject: [PATCH 32/32] Inline default as with other options --- lib/mrsk/configuration.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index aa4cd67e..d6cf09f1 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -12,8 +12,6 @@ class Mrsk::Configuration attr_accessor :destination attr_accessor :raw_config - MAX_LOG_SIZE = "10m" - class << self def create_from(config_file:, destination: nil, version: nil) raw_config = load_config_files(config_file, *destination_config_file(config_file, destination)) @@ -128,7 +126,7 @@ class Mrsk::Configuration optionize({ "log-driver" => raw_config.logging["driver"] }.compact) + argumentize("--log-opt", raw_config.logging["options"]) else - argumentize("--log-opt", { "max-size" => MAX_LOG_SIZE }) + argumentize("--log-opt", { "max-size" => "10m" }) end end