From 2746a48e88209f866af0c1f1fcaab9028c3c4fb0 Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Thu, 22 Jun 2023 15:13:47 +0400 Subject: [PATCH 1/9] Login to the registry proactively before stoping Accessory and Traefik --- lib/mrsk/cli/accessory.rb | 10 +++++++--- lib/mrsk/cli/traefik.rb | 14 +++++++++++--- test/cli/accessory_test.rb | 3 ++- test/cli/traefik_test.rb | 3 ++- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/lib/mrsk/cli/accessory.rb b/lib/mrsk/cli/accessory.rb index 6d80e115..f597a2a6 100644 --- a/lib/mrsk/cli/accessory.rb +++ b/lib/mrsk/cli/accessory.rb @@ -1,6 +1,6 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base desc "boot [NAME]", "Boot new accessory service on host (use NAME=all to boot all accessories)" - def boot(name) + def boot(name, login: true) mutating do if name == "all" MRSK.accessory_names.each { |accessory_name| boot(accessory_name) } @@ -10,7 +10,7 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base upload(name) on(accessory.hosts) do - execute *MRSK.registry.login + execute *MRSK.registry.login if login execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug execute *accessory.run end @@ -53,9 +53,13 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base def reboot(name) mutating do with_accessory(name) do |accessory| + on(accessory.hosts) do + execute *MRSK.registry.login + end + stop(name) remove_container(name) - boot(name) + boot(name, login: false) end end end diff --git a/lib/mrsk/cli/traefik.rb b/lib/mrsk/cli/traefik.rb index 30fd10f3..c8ad77f1 100644 --- a/lib/mrsk/cli/traefik.rb +++ b/lib/mrsk/cli/traefik.rb @@ -1,9 +1,9 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base desc "boot", "Boot Traefik on servers" - def boot + def boot(login: true) mutating do on(MRSK.traefik_hosts) do - execute *MRSK.registry.login + execute *MRSK.registry.login if login execute *MRSK.traefik.run, raise_on_non_zero_exit: false end end @@ -12,9 +12,17 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base desc "reboot", "Reboot Traefik on servers (stop container, remove container, start new container)" def reboot mutating do + on(MRSK.traefik_hosts) do + execute *MRSK.registry.login + end + stop remove_container - boot + boot(login: false) + + on(MRSK.traefik_hosts) do + execute *MRSK.traefik.run, raise_on_non_zero_exit: false + end end end diff --git a/test/cli/accessory_test.rb b/test/cli/accessory_test.rb index 28c4977f..4f3210bb 100644 --- a/test/cli/accessory_test.rb +++ b/test/cli/accessory_test.rb @@ -40,9 +40,10 @@ class CliAccessoryTest < CliTestCase end test "reboot" do + Mrsk::Commands::Registry.any_instance.expects(:login) Mrsk::Cli::Accessory.any_instance.expects(:stop).with("mysql") Mrsk::Cli::Accessory.any_instance.expects(:remove_container).with("mysql") - Mrsk::Cli::Accessory.any_instance.expects(:boot).with("mysql") + Mrsk::Cli::Accessory.any_instance.expects(:boot).with("mysql", login: false) run_command("reboot", "mysql") end diff --git a/test/cli/traefik_test.rb b/test/cli/traefik_test.rb index a1f661d2..450181bb 100644 --- a/test/cli/traefik_test.rb +++ b/test/cli/traefik_test.rb @@ -9,9 +9,10 @@ class CliTraefikTest < CliTestCase end test "reboot" do + Mrsk::Commands::Registry.any_instance.expects(:login).twice Mrsk::Cli::Traefik.any_instance.expects(:stop) Mrsk::Cli::Traefik.any_instance.expects(:remove_container) - Mrsk::Cli::Traefik.any_instance.expects(:boot) + Mrsk::Cli::Traefik.any_instance.expects(:boot).with(login: false) run_command("reboot") end From a3cc2317e24a390bc2d7bf10bc70be2c7fce2cdd Mon Sep 17 00:00:00 2001 From: Jonas Helgemo Date: Mon, 26 Jun 2023 11:57:23 +0200 Subject: [PATCH 2/9] Updated README.md to make setup examples consistent - SSH and apt examples should use same username --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f148a5f2..d0f73d65 100644 --- a/README.md +++ b/README.md @@ -206,13 +206,13 @@ ssh: user: app ``` -If you are using non-root user, you need to bootstrap your servers manually, before using them with MRSK. On Ubuntu, you'd do: +If you are using non-root user (`app` as above example), you need to bootstrap your servers manually, before using them with MRSK. On Ubuntu, you'd do: ```bash sudo apt update sudo apt upgrade -y sudo apt install -y docker.io curl git -sudo usermod -a -G docker ubuntu +sudo usermod -a -G docker app ``` ### Using a proxy SSH host From f8d651af0d1fe929b51879fcda75360f84a8e305 Mon Sep 17 00:00:00 2001 From: Nick Hammond Date: Wed, 28 Jun 2023 10:03:27 -0600 Subject: [PATCH 3/9] Update README.md Add a note about utilizing biometrics with the envify example. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f148a5f2..3fdbb46b 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,8 @@ This template can safely be checked into git. Then everyone deploying the app ca If you need separate env variables for different destinations, you can set them with `.env.destination.erb` for the template, which will generate `.env.staging` when run with `mrsk envify -d staging`. +Note: If you utilize biometrics with 1Password you can remove the `session_token` related parts in the example and just call `op read op://Vault/Docker Hub/password -n`. + #### Bitwarden as a secret store If you are using open source secret store like bitwarden, you can create `.env.erb` as a template which looks up the secrets. From e760cfa4574b5c98815abbc25846d303a56d521e Mon Sep 17 00:00:00 2001 From: Nick Hammond Date: Sat, 8 Jul 2023 10:59:55 -0600 Subject: [PATCH 4/9] Minor tweaks to hooks section in readme --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f148a5f2..1435e27c 100644 --- a/README.md +++ b/README.md @@ -922,8 +922,8 @@ firing a JSON webhook. These variables include: - `MRSK_RECORDED_AT` - UTC timestamp in ISO 8601 format, e.g. `2023-04-14T17:07:31Z` - `MRSK_PERFORMER` - the local user performing the command (from `whoami`) - `MRSK_SERVICE_VERSION` - an abbreviated service and version for use in messages, e.g. app@150b24f -- `MRSK_VERSION` - an full version being deployed -- `MRSK_HOSTS` - a comma separated list of the hosts targeted by the command +- `MRSK_VERSION` - the full version being deployed +- `MRSK_HOSTS` - a comma-separated list of the hosts targeted by the command - `MRSK_COMMAND` - The command we are running - `MRSK_SUBCOMMAND` - optional: The subcommand we are running - `MRSK_DESTINATION` - optional: destination, e.g. "staging" @@ -940,9 +940,8 @@ Used for pre-build checks - e.g. there are no uncommitted changes or that CI has 3. pre-deploy For final checks before deploying, e.g. checking CI completed -3. post-deploy - run after a deploy, redeploy or rollback - -This hook is also passed a `MRSK_RUNTIME` env variable. +3. post-deploy - run after a deploy, redeploy or rollback. +This hook is also passed a `MRSK_RUNTIME` env variable set to the total seconds the deploy took. This could be used to broadcast a deployment message, or register the new version with an APM. @@ -953,7 +952,7 @@ The command could look something like: curl -q -d content="[My App] ${MRSK_PERFORMER} Rolled back to version ${MRSK_VERSION}" https://3.basecamp.com/XXXXX/integrations/XXXXX/buckets/XXXXX/chats/XXXXX/lines ``` -That'll post a line like follows to a preconfigured chatbot in Basecamp: +That'll post a line like the following to a preconfigured chatbot in Basecamp: ``` [My App] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de From da1c049829fb69239336080c4ed38f75d0329d07 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 14 Jul 2023 12:42:54 +0100 Subject: [PATCH 5/9] Add registry container output to debug --- test/integration/integration_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/integration_test.rb b/test/integration/integration_test.rb index 821497f5..00e73d62 100644 --- a/test/integration/integration_test.rb +++ b/test/integration/integration_test.rb @@ -11,7 +11,7 @@ class IntegrationTest < ActiveSupport::TestCase teardown do unless passed? - [:deployer, :vm1, :vm2, :shared, :load_balancer].each do |container| + [:deployer, :vm1, :vm2, :shared, :load_balancer, :registry].each do |container| puts puts "Logs for #{container}:" docker_compose :logs, container From cd88c49c42355a4eca0d06f440781e239b47a269 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 14 Jul 2023 16:08:47 +0100 Subject: [PATCH 6/9] Configurable SSH log levels Allow ssh log_level to be set to debug connection issues. --- README.md | 9 +++++++++ lib/mrsk/configuration.rb | 13 +++++++++++-- test/configuration_test.rb | 27 +++++++++++++++++++++++---- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 67401068..fa5ad6e1 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,15 @@ ssh: proxy_command: aws ssm start-session --target %h --document-name AWS-StartSSHSession --parameters 'portNumber=%p' --region=us-east-1 ## ssh via aws ssm ``` +### Configuring the SSH log level + +```yaml +ssh: + log_level: debug +``` + +Valid levels are `debug`, `info`, `warn`, `error` and `fatal` (default). + ### Using env variables You can inject env variables into the app containers using `env`: diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index ee6dce05..e570de55 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -153,7 +153,15 @@ class Mrsk::Configuration end def ssh_options - { user: ssh_user, proxy: ssh_proxy, auth_methods: [ "publickey" ] }.compact + { user: ssh_user, proxy: ssh_proxy, auth_methods: [ "publickey" ], logger: ssh_logger }.compact + end + + def ssh_logger + @ssh_logger ||= ::Logger.new(STDERR).tap { |logger| logger.level = ssh_log_level } + end + + def ssh_log_level + (raw_config.ssh && raw_config.ssh["log_level"]) || ::Logger::FATAL end @@ -185,7 +193,8 @@ class Mrsk::Configuration service_with_version: service_with_version, env_args: env_args, volume_args: volume_args, - ssh_options: ssh_options, + ssh_options: ssh_options.except(:logger), + ssh_log_level: ssh_log_level, builder: builder.to_h, accessories: raw_config.accessories, logging: logging_args, diff --git a/test/configuration_test.rb b/test/configuration_test.rb index 0f7e12ff..3d362293 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -211,17 +211,21 @@ class ConfigurationTest < ActiveSupport::TestCase assert_equal "root", @config.ssh_options[:user] config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "user" => "app" }) }) - assert_equal "app", @config.ssh_options[:user] + assert_equal "app", config.ssh_options[:user] + assert_equal 4, config.ssh_options[:logger].level + + config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "log_level" => "debug" }) }) + assert_equal 0, config.ssh_options[:logger].level end test "ssh options with proxy host" do config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "proxy" => "1.2.3.4" }) }) - assert_equal "root@1.2.3.4", @config.ssh_options[:proxy].jump_proxies + assert_equal "root@1.2.3.4", config.ssh_options[:proxy].jump_proxies end test "ssh options with proxy host and user" do config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "proxy" => "app@1.2.3.4" }) }) - assert_equal "app@1.2.3.4", @config.ssh_options[:proxy].jump_proxies + assert_equal "app@1.2.3.4", config.ssh_options[:proxy].jump_proxies end test "volume_args" do @@ -266,7 +270,22 @@ class ConfigurationTest < ActiveSupport::TestCase end test "to_h" do - assert_equal({ :roles=>["web"], :hosts=>["1.1.1.1", "1.1.1.2"], :primary_host=>"1.1.1.1", :version=>"missing", :repository=>"dhh/app", :absolute_image=>"dhh/app:missing", :service_with_version=>"app-missing", :env_args=>["-e", "REDIS_URL=\"redis://x/y\""], :ssh_options=>{:user=>"root", :auth_methods=>["publickey"]}, :volume_args=>["--volume", "/local/path:/container/path"], :builder=>{}, :logging=>["--log-opt", "max-size=\"10m\""], :healthcheck=>{"path"=>"/up", "port"=>3000, "max_attempts" => 7 }}, @config.to_h) + assert_equal({ + :roles=>["web"], + :hosts=>["1.1.1.1", "1.1.1.2"], + :primary_host=>"1.1.1.1", + :version=>"missing", + :repository=>"dhh/app", + :absolute_image=>"dhh/app:missing", + :service_with_version=>"app-missing", + :env_args=>["-e", "REDIS_URL=\"redis://x/y\""], + :ssh_options=>{:user=>"root", :auth_methods=>["publickey"]}, + :ssh_log_level=>4, + :volume_args=>["--volume", "/local/path:/container/path"], + :builder=>{}, + :logging=>["--log-opt", "max-size=\"10m\""], + :healthcheck=>{"path"=>"/up", "port"=>3000, "max_attempts" => 7 } + }, @config.to_h) end test "min version is lower" do From e6ca2705379dc66d15ee098110d66a474ab2d65e Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Sat, 15 Jul 2023 21:50:39 +0400 Subject: [PATCH 7/9] Include service name to lock details --- lib/mrsk/commands/lock.rb | 2 +- test/cli/cli_test_case.rb | 4 ++-- test/cli/main_test.rb | 8 ++++---- test/commands/lock_test.rb | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/mrsk/commands/lock.rb b/lib/mrsk/commands/lock.rb index 6f84a5cc..6166d486 100644 --- a/lib/mrsk/commands/lock.rb +++ b/lib/mrsk/commands/lock.rb @@ -40,7 +40,7 @@ class Mrsk::Commands::Lock < Mrsk::Commands::Base end def lock_dir - :mrsk_lock + "mrsk_lock-#{config.service}" end def lock_details_file diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb index 05b0ff9f..f094378d 100644 --- a/test/cli/cli_test_case.rb +++ b/test/cli/cli_test_case.rb @@ -29,9 +29,9 @@ class CliTestCase < ActiveSupport::TestCase def stub_locking SSHKit::Backend::Abstract.any_instance.stubs(:execute) - .with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock } + .with { |arg1, arg2| arg1 == :mkdir && arg2 == "mrsk_lock-app" } SSHKit::Backend::Abstract.any_instance.stubs(:execute) - .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" } + .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock-app/details" } end def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: nil) diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index 728ce24e..730212ec 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -63,11 +63,11 @@ class CliMainTest < CliTestCase Thread.report_on_exception = false SSHKit::Backend::Abstract.any_instance.stubs(:execute) - .with { |*arg| arg[0..1] == [:mkdir, :mrsk_lock] } - .raises(RuntimeError, "mkdir: cannot create directory ‘mrsk_lock’: File exists") + .with { |*arg| arg[0..1] == [:mkdir, 'mrsk_lock-app'] } + .raises(RuntimeError, "mkdir: cannot create directory ‘mrsk_lock-app’: File exists") SSHKit::Backend::Abstract.any_instance.expects(:capture_with_debug) - .with(:stat, :mrsk_lock, ">", "/dev/null", "&&", :cat, "mrsk_lock/details", "|", :base64, "-d") + .with(:stat, 'mrsk_lock-app', ">", "/dev/null", "&&", :cat, "mrsk_lock-app/details", "|", :base64, "-d") assert_raises(Mrsk::Cli::LockError) do run_command("deploy") @@ -78,7 +78,7 @@ class CliMainTest < CliTestCase Thread.report_on_exception = false SSHKit::Backend::Abstract.any_instance.stubs(:execute) - .with { |*arg| arg[0..1] == [:mkdir, :mrsk_lock] } + .with { |*arg| arg[0..1] == [:mkdir, 'mrsk_lock-app'] } .raises(SocketError, "getaddrinfo: nodename nor servname provided, or not known") assert_raises(SSHKit::Runner::ExecuteError) do diff --git a/test/commands/lock_test.rb b/test/commands/lock_test.rb index bfb76597..f32aac07 100644 --- a/test/commands/lock_test.rb +++ b/test/commands/lock_test.rb @@ -10,19 +10,19 @@ class CommandsLockTest < ActiveSupport::TestCase test "status" do assert_equal \ - "stat mrsk_lock > /dev/null && cat mrsk_lock/details | base64 -d", + "stat mrsk_lock-app > /dev/null && cat mrsk_lock-app/details | base64 -d", new_command.status.join(" ") end test "acquire" do assert_match \ - /mkdir mrsk_lock && echo ".*" > mrsk_lock\/details/m, + /mkdir mrsk_lock-app && echo ".*" > mrsk_lock-app\/details/m, new_command.acquire("Hello", "123").join(" ") end test "release" do assert_match \ - "rm mrsk_lock/details && rm -r mrsk_lock", + "rm mrsk_lock-app/details && rm -r mrsk_lock-app", new_command.release.join(" ") end From 83f5f3f053fd9abe98921dfe7c6b75399bc9b817 Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Mon, 17 Jul 2023 10:39:50 +0400 Subject: [PATCH 8/9] Updated README with locking directory name --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 67401068..feedf518 100644 --- a/README.md +++ b/README.md @@ -868,7 +868,7 @@ If you wish to remove the entire application, including Traefik, containers, ima ## Locking -Commands that are unsafe to run concurrently will take a deploy lock while they run. The lock is the `mrsk_lock` directory on the primary server. +Commands that are unsafe to run concurrently will take a deploy lock while they run. The lock is the `mrsk_lock-` directory on the primary server. You can check the lock status with: From a29e188c901018aff6b58e100793b363ca1a5f5f Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Wed, 19 Jul 2023 15:08:42 +0400 Subject: [PATCH 9/9] Removed not needed MRSK.traefik.run command in Traefil reboot --- lib/mrsk/cli/traefik.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/mrsk/cli/traefik.rb b/lib/mrsk/cli/traefik.rb index c8ad77f1..95707789 100644 --- a/lib/mrsk/cli/traefik.rb +++ b/lib/mrsk/cli/traefik.rb @@ -19,10 +19,6 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base stop remove_container boot(login: false) - - on(MRSK.traefik_hosts) do - execute *MRSK.traefik.run, raise_on_non_zero_exit: false - end end end