diff --git a/README.md b/README.md index 9203cac2..ca550b15 100644 --- a/README.md +++ b/README.md @@ -668,43 +668,6 @@ servers: This assumes the Cron settings are stored in `config/crontab`. -### Using audit broadcasts - -If you'd like to broadcast audits of deploys, rollbacks, etc to a chatroom or elsewhere, you can configure the `audit_broadcast_cmd` setting with the path to a bin file that will be passed the audit line as the first argument: - -```yaml -audit_broadcast_cmd: - bin/audit_broadcast -``` - -The broadcast command could look something like: - -```bash -#!/usr/bin/env bash -curl -q -d content="[My App] ${1}" https://3.basecamp.com/XXXXX/integrations/XXXXX/buckets/XXXXX/chats/XXXXX/lines -``` - -That'll post a line like follows to a preconfigured chatbot in Basecamp: - -``` -[My App] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de -``` - -`MRSK_*` environment variables are available to the broadcast command for -fine-grained audit reporting, e.g. for triggering deployment reports or -firing a JSON webhook. These variables include: -- `MRSK_RECORDED_AT` - UTC timestamp in ISO 8601 format, e.g. `2023-04-14T17:07:31Z` -- `MRSK_PERFORMER` - the local user performing the command (from `whoami`) -- `MRSK_MESSAGE` - the full audit message, e.g. "Deployed app@150b24f" -- `MRSK_DESTINATION` - optional: destination, e.g. "staging" -- `MRSK_ROLE` - optional: role targeted, e.g. "web" - -Use `mrsk broadcast` to test and troubleshoot your broadcast command: - -```bash -mrsk broadcast -m "test audit message" -``` - ### Healthcheck MRSK uses Docker healtchecks to check the health of your application during deployment. Traefik uses this same healthcheck status to determine when a container is ready to receive traffic. @@ -902,6 +865,52 @@ When `limit` is specified, containers will be booted on, at most, `limit` hosts These settings only apply when booting containers (using `mrsk deploy`, or `mrsk app boot`). For other commands, MRSK continues to run commands in parallel across all hosts. +## Hooks + +You can run custom scripts at specific points with hooks. + +Hooks should be stored in the .mrsk/hooks folder. Running mrsk init will build that folder and add some sample scripts. + +You can change their location by setting `hooks_path` in the configuration file. + +If the script returns a non-zero exit code the command will be aborted. + +`MRSK_*` environment variables are available to the hooks command for +fine-grained audit reporting, e.g. for triggering deployment reports or +firing a JSON webhook. These variables include: +- `MRSK_RECORDED_AT` - UTC timestamp in ISO 8601 format, e.g. `2023-04-14T17:07:31Z` +- `MRSK_PERFORMER` - the local user performing the command (from `whoami`) +- `MRSK_MESSAGE` - the full audit message, e.g. "Deployed app@150b24f" +- `MRSK_SERVICE_VERSION` - an abbreviated version (for use in messages) +- `MRSK_DESTINATION` - optional: destination, e.g. "staging" +- `MRSK_ROLE` - optional: role targeted, e.g. "web" + +There are two hooks: + +1. pre-build +Used for pre-build checks - e.g. there are no uncommitted changes or that CI has passed. + +2. post-deploy - run after a deploy, redeploy or rollback + +This hook is also passed a `MRSK_RUNTIME` env variable. + +This could be used to broadcast a deployment message, or register the new version with an APM. + +The command could look something like: + +```bash +#!/usr/bin/env bash +curl -q -d content="[My App] ${MRSK_PERFORMER} Rolled back to version ${MRSK_VERSION}" https://3.basecamp.com/XXXXX/integrations/XXXXX/buckets/XXXXX/chats/XXXXX/lines +``` + +That'll post a line like follows to a preconfigured chatbot in Basecamp: + +``` +[My App] [dhh] Rolled back to version d264c4e92470ad1bd18590f04466787262f605de +``` + +Set `--skip_hooks` to avoid running the hooks. + ## Stage of development This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com). diff --git a/lib/mrsk/cli.rb b/lib/mrsk/cli.rb index c974a814..bc541d06 100644 --- a/lib/mrsk/cli.rb +++ b/lib/mrsk/cli.rb @@ -1,5 +1,6 @@ module Mrsk::Cli class LockError < StandardError; end + class HookError < StandardError; end end # SSHKit uses instance eval, so we need a global const for ergonomics diff --git a/lib/mrsk/cli/accessory.rb b/lib/mrsk/cli/accessory.rb index c34a3902..1343a5f2 100644 --- a/lib/mrsk/cli/accessory.rb +++ b/lib/mrsk/cli/accessory.rb @@ -14,8 +14,6 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug execute *accessory.run end - - audit_broadcast "Booted accessory #{name}" unless options[:skip_broadcast] end end end diff --git a/lib/mrsk/cli/base.rb b/lib/mrsk/cli/base.rb index 7a261dfe..395df4cc 100644 --- a/lib/mrsk/cli/base.rb +++ b/lib/mrsk/cli/base.rb @@ -20,7 +20,7 @@ module Mrsk::Cli class_option :config_file, aliases: "-c", default: "config/deploy.yml", desc: "Path to config file" class_option :destination, aliases: "-d", desc: "Specify destination to be used for config file (staging -> deploy.staging.yml)" - class_option :skip_broadcast, aliases: "-B", type: :boolean, default: false, desc: "Skip audit broadcasts" + class_option :skip_hooks, aliases: "-H", type: :boolean, default: false, desc: "Don't run hooks" def initialize(*) super @@ -72,10 +72,6 @@ module Mrsk::Cli puts " Finished all in #{sprintf("%.1f seconds", runtime)}" end - def audit_broadcast(line) - run_locally { execute *MRSK.auditor.broadcast(line), verbosity: :debug } - end - def with_lock if MRSK.holding_lock? yield @@ -134,5 +130,16 @@ module Mrsk::Cli MRSK.hold_lock_on_error = false end end + + def run_hook(hook, **details) + if !options[:skip_hooks] && MRSK.hook.hook_exists?(hook) + say "Running the #{hook} hook...", :magenta + run_locally do + MRSK.with_verbosity(:debug) { execute *MRSK.hook.run(hook, **details) } + rescue SSHKit::Command::Failed + raise HookError.new("Hook `#{hook}` failed") + end + end + end end end diff --git a/lib/mrsk/cli/build.rb b/lib/mrsk/cli/build.rb index 0e6e9a7c..9628c757 100644 --- a/lib/mrsk/cli/build.rb +++ b/lib/mrsk/cli/build.rb @@ -14,11 +14,12 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base with_lock do cli = self + verify_local_dependencies + run_hook "pre-build" + run_locally do begin - if cli.verify_local_dependencies - MRSK.with_verbosity(:debug) { execute *MRSK.builder.push } - end + MRSK.with_verbosity(:debug) { execute *MRSK.builder.push } rescue SSHKit::Command::Failed => e if e.message =~ /(no builder)|(no such file or directory)/ error "Missing compatible builder, so creating a new one first" @@ -82,21 +83,18 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base end end + private + def verify_local_dependencies + run_locally do + begin + execute *MRSK.builder.ensure_local_dependencies_installed + rescue SSHKit::Command::Failed => e + build_error = e.message =~ /command not found/ ? + "Docker is not installed locally" : + "Docker buildx plugin is not installed locally" - desc "", "" # Really a private method, but needed to be invoked from #push - def verify_local_dependencies - run_locally do - begin - execute *MRSK.builder.ensure_local_dependencies_installed - rescue SSHKit::Command::Failed => e - build_error = e.message =~ /command not found/ ? - "Docker is not installed locally" : - "Docker buildx plugin is not installed locally" - - raise BuildError, build_error + raise BuildError, build_error + end end end - - true - end end diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index a3d129d2..8f078bdc 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -44,7 +44,7 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base end end - audit_broadcast "Deployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast] + run_hook "post-deploy", runtime: runtime.round end desc "redeploy", "Deploy app to servers without bootstrapping servers, starting Traefik, pruning, and registry login" @@ -72,25 +72,29 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base end end - audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast] + run_hook "post-deploy", runtime: runtime.round end desc "rollback [VERSION]", "Rollback app to VERSION" def rollback(version) - with_lock do - invoke_options = deploy_options + rolled_back = false + runtime = print_runtime do + with_lock do + invoke_options = deploy_options - MRSK.config.version = version - old_version = nil + MRSK.config.version = version + old_version = nil - if container_available?(version) - invoke "mrsk:cli:app:boot", [], invoke_options.merge(version: version) - - audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast] - else - say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red + if container_available?(version) + invoke "mrsk:cli:app:boot", [], invoke_options.merge(version: version) + rolled_back = true + else + say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red + end end end + + run_hook "post-deploy", runtime: runtime.round if rolled_back end desc "details", "Show details about all containers" @@ -132,6 +136,14 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base puts "Created .env file" end + unless (hooks_dir = Pathname.new(File.expand_path(".mrsk/hooks"))).exist? + hooks_dir.mkpath + Pathname.new(File.expand_path("templates/sample_hooks", __dir__)).each_child do |sample_hook| + FileUtils.cp sample_hook, hooks_dir, preserve: true + end + puts "Created sample hooks in .mrsk/hooks" + end + if options[:bundle] if (binstub = Pathname.new(File.expand_path("bin/mrsk"))).exist? puts "Binstub already exists in bin/mrsk (remove first to create a new one)" @@ -172,13 +184,6 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base end end - desc "broadcast", "Broadcast an audit message" - option :message, aliases: "-m", type: :string, desc: "Audit message", required: true - def broadcast - say "Broadcast: #{options[:message]}", :magenta - audit_broadcast options[:message] - end - desc "version", "Show MRSK version" def version puts Mrsk::VERSION @@ -235,8 +240,4 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base def deploy_options { "version" => MRSK.config.version }.merge(options.without("skip_push")) end - - def service_version(version = MRSK.config.abbreviated_version) - [ MRSK.config.service, version ].compact.join("@") - end end diff --git a/lib/mrsk/cli/templates/deploy.yml b/lib/mrsk/cli/templates/deploy.yml index 7d710632..45b987f5 100644 --- a/lib/mrsk/cli/templates/deploy.yml +++ b/lib/mrsk/cli/templates/deploy.yml @@ -25,10 +25,6 @@ registry: # secret: # - RAILS_MASTER_KEY -# Call a broadcast command on deploys. -# audit_broadcast_cmd: -# bin/broadcast_to_bc - # Use a different ssh user than root # ssh: # user: app diff --git a/lib/mrsk/cli/templates/sample_hooks/post-deploy.sample b/lib/mrsk/cli/templates/sample_hooks/post-deploy.sample new file mode 100755 index 00000000..abc18ccb --- /dev/null +++ b/lib/mrsk/cli/templates/sample_hooks/post-deploy.sample @@ -0,0 +1,18 @@ +#!/bin/sh + +# A sample post-deploy hook +# +# Checks: +# 1. We have a clean checkout +# 2. A remote is configured +# 3. The branch has been pushed to the remote +# 4. The version we are deploying matches the remote +# +# These environment variables are available: +# MRSK_RECORDED_AT +# MRSK_PERFORMER +# MRSK_VERSION +# MRSK_DESTINATION (if set) +# MRSK_RUNTIME + +echo "$MRSK_PERFORMER deployed $MRSK_VERSION to $MRSK_DESTINATION in $MRSK_RUNTIME seconds" diff --git a/lib/mrsk/cli/templates/sample_hooks/pre-build.sample b/lib/mrsk/cli/templates/sample_hooks/pre-build.sample new file mode 100755 index 00000000..0d12be7e --- /dev/null +++ b/lib/mrsk/cli/templates/sample_hooks/pre-build.sample @@ -0,0 +1,49 @@ +#!/bin/sh + +# A sample pre-build hook +# +# Checks: +# 1. We have a clean checkout +# 2. A remote is configured +# 3. The branch has been pushed to the remote +# 4. The version we are deploying matches the remote +# +# These environment variables are available: +# MRSK_RECORDED_AT +# MRSK_PERFORMER +# MRSK_VERSION +# MRSK_DESTINATION (if set) + +if [ -n "$(git status --porcelain)" ]; then + echo "Git checkout is not clean, aborting..." >&2 + git status --porcelain >&2 + exit 1 +fi + +first_remote=$(git remote) + +if [ -z "$first_remote" ]; then + echo "No git remote set, aborting..." >&2 + exit 1 +fi + +current_branch=$(git branch --show-current) + +if [ -z "$current_branch" ]; then + echo "No git remote set, aborting..." >&2 + exit 1 +fi + +remote_head=$(git ls-remote $first_remote --tags $current_branch | cut -f1) + +if [ -z "$remote_head" ]; then + echo "Branch not pushed to remote, aborting..." >&2 + exit 1 +fi + +if [ "$MRSK_VERSION" != "$remote_head" ]; then + echo "Version ($MRSK_VERSION) does not match remote HEAD ($remote_head), aborting..." >&2 + exit 1 +fi + +exit 0 diff --git a/lib/mrsk/commander.rb b/lib/mrsk/commander.rb index 4ef45f0a..bdfef962 100644 --- a/lib/mrsk/commander.rb +++ b/lib/mrsk/commander.rb @@ -100,6 +100,14 @@ class Mrsk::Commander @healthcheck ||= Mrsk::Commands::Healthcheck.new(config) end + def hook + @hook ||= Mrsk::Commands::Hook.new(config) + end + + def lock + @lock ||= Mrsk::Commands::Lock.new(config) + end + def prune @prune ||= Mrsk::Commands::Prune.new(config) end @@ -112,10 +120,6 @@ class Mrsk::Commander @traefik ||= Mrsk::Commands::Traefik.new(config) end - def lock - @lock ||= Mrsk::Commands::Lock.new(config) - end - def with_verbosity(level) old_level = self.verbosity diff --git a/lib/mrsk/commands/auditor.rb b/lib/mrsk/commands/auditor.rb index 69ee03cd..5089fb4d 100644 --- a/lib/mrsk/commands/auditor.rb +++ b/lib/mrsk/commands/auditor.rb @@ -1,27 +1,18 @@ -require "time" - class Mrsk::Commands::Auditor < Mrsk::Commands::Base attr_reader :details def initialize(config, **details) super(config) - @details = default_details.merge(details) + @details = details end # Runs remotely def record(line, **details) append \ - [ :echo, *audit_tags(**details), line ], + [ :echo, audit_tags(**details).except(:version, :service_version).to_s, line ], audit_log_file end - # Runs locally - def broadcast(line, **details) - if broadcast_cmd = config.audit_broadcast_cmd - [ broadcast_cmd, *broadcast_args(line, **details), env: env_for(event: line, **details) ] - end - end - def reveal [ :tail, "-n", 50, audit_log_file ] end @@ -31,29 +22,7 @@ class Mrsk::Commands::Auditor < Mrsk::Commands::Base [ "mrsk", config.service, config.destination, "audit.log" ].compact.join("-") end - def default_details - { recorded_at: Time.now.utc.iso8601, - performer: `whoami`.chomp, - destination: config.destination } - end - def audit_tags(**details) - tags_for **self.details.merge(details) - end - - def broadcast_args(line, **details) - "'#{broadcast_tags(**details).join(" ")} #{line}'" - end - - def broadcast_tags(**details) - tags_for **self.details.merge(details).except(:recorded_at) - end - - def tags_for(**details) - details.compact.values.map { |value| "[#{value}]" } - end - - def env_for(**details) - self.details.merge(details).compact.transform_keys { |detail| "MRSK_#{detail.upcase}" } + tags(**self.details, **details) end end diff --git a/lib/mrsk/commands/base.rb b/lib/mrsk/commands/base.rb index 81c8420c..391c25f2 100644 --- a/lib/mrsk/commands/base.rb +++ b/lib/mrsk/commands/base.rb @@ -53,5 +53,9 @@ module Mrsk::Commands def docker(*args) args.compact.unshift :docker end + + def tags(**details) + Mrsk::Tags.from_config(config, **details) + end end end diff --git a/lib/mrsk/commands/hook.rb b/lib/mrsk/commands/hook.rb new file mode 100644 index 00000000..912d4480 --- /dev/null +++ b/lib/mrsk/commands/hook.rb @@ -0,0 +1,14 @@ +class Mrsk::Commands::Hook < Mrsk::Commands::Base + def run(hook, **details) + [ hook_file(hook), env: tags(**details).env ] + end + + def hook_exists?(hook) + Pathname.new(hook_file(hook)).exist? + end + + private + def hook_file(hook) + "#{config.hooks_path}/#{hook}" + end +end diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index f4837079..e78e45cb 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -6,7 +6,7 @@ require "erb" require "net/ssh/proxy/jump" class Mrsk::Configuration - delegate :service, :image, :servers, :env, :labels, :registry, :builder, :stop_wait_time, to: :raw_config, allow_nil: true + delegate :service, :image, :servers, :env, :labels, :registry, :builder, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Mrsk::Utils attr_accessor :destination @@ -157,10 +157,6 @@ class Mrsk::Configuration end - def audit_broadcast_cmd - raw_config.audit_broadcast_cmd - end - def healthcheck { "path" => "/up", "port" => 3000, "max_attempts" => 7 }.merge(raw_config.healthcheck || {}) end @@ -197,6 +193,10 @@ class Mrsk::Configuration raw_config.traefik || {} end + def hooks_path + raw_config.hooks_path || ".mrsk/hooks" + end + private # Will raise ArgumentError if any required config keys are missing def ensure_required_keys_present diff --git a/lib/mrsk/tags.rb b/lib/mrsk/tags.rb new file mode 100644 index 00000000..c134a8f7 --- /dev/null +++ b/lib/mrsk/tags.rb @@ -0,0 +1,39 @@ +require "time" + +class Mrsk::Tags + attr_reader :config, :tags + + class << self + def from_config(config, **extra) + new(**default_tags(config), **extra) + end + + def default_tags(config) + { recorded_at: Time.now.utc.iso8601, + performer: `whoami`.chomp, + destination: config.destination, + version: config.version, + service_version: service_version(config) } + end + + def service_version(config) + [ config.service, config.abbreviated_version ].compact.join("@") + end + end + + def initialize(**tags) + @tags = tags.compact + end + + def env + tags.transform_keys { |detail| "MRSK_#{detail.upcase}" } + end + + def to_s + tags.values.map { |value| "[#{value}]" }.join(" ") + end + + def except(*tags) + self.class.new(**self.tags.except(*tags)) + end +end diff --git a/lib/mrsk/utils.rb b/lib/mrsk/utils.rb index 62dc6d8d..e463f547 100644 --- a/lib/mrsk/utils.rb +++ b/lib/mrsk/utils.rb @@ -84,6 +84,13 @@ module Mrsk::Utils # Abbreviate a git revhash for concise display def abbreviate_version(version) - version[0...7] if version + if version + # Don't abbreviate _uncommitted_ + if version.include?("_") + version + else + version[0...7] + end + end end end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index f198da3a..806c8afe 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -2,12 +2,7 @@ require_relative "cli_test_case" class CliAppTest < CliTestCase test "boot" do - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version - - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") - .returns("running") # health check - + stub_running run_command("boot").tap do |output| assert_match "docker tag dhh/app:latest dhh/app:latest", output assert_match "docker run --detach --restart unless-stopped", output @@ -51,7 +46,7 @@ class CliAppTest < CliTestCase end test "boot errors leave lock in place" do - invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" } + invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" } Mrsk::Cli::App.any_instance.expects(:using_version).raises(RuntimeError) @@ -183,4 +178,12 @@ class CliAppTest < CliTestCase def run_command(*command, config: :with_accessories) stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_#{config}.yml", "--hosts", "1.1.1.1"]) } end + + def stub_running + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("running") # health check + end end diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb index 0835262c..ece83851 100644 --- a/test/cli/build_test.rb +++ b/test/cli/build_test.rb @@ -9,17 +9,19 @@ class CliBuildTest < CliTestCase end test "push" do - Mrsk::Cli::Build.any_instance.stubs(:verify_local_dependencies).returns(true) run_command("push").tap do |output| + assert_match /docker --version && docker buildx version/, output assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder mrsk-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output end end test "push without builder" do stub_locking - Mrsk::Cli::Build.any_instance.stubs(:verify_local_dependencies).returns(true) SSHKit::Backend::Abstract.any_instance.stubs(:execute) - .with { |arg| arg == :docker } + .with(:docker, "--version", "&&", :docker, :buildx, "version") + + SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with { |*args| args[0..1] == [:docker, :buildx] } .raises(SSHKit::Command::Failed.new("no builder")) .then .returns(true) @@ -29,6 +31,24 @@ class CliBuildTest < CliTestCase end end + test "push with no buildx plugin" do + stub_locking + SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with(:docker, "--version", "&&", :docker, :buildx, "version") + .raises(SSHKit::Command::Failed.new("no buildx")) + + Mrsk::Commands::Builder.any_instance.stubs(:native_and_local?).returns(false) + assert_raises(Mrsk::Cli::Build::BuildError) { run_command("push") } + end + + test "push pre-build hook failure" do + fail_hook("pre-build") + + assert_raises(Mrsk::Cli::HookError) { run_command("push") } + + assert @executions.none? { |args| args[0..2] == [:docker, :buildx, :build] } + end + test "pull" do run_command("pull").tap do |output| assert_match /docker image rm --force dhh\/app:999/, output @@ -70,32 +90,15 @@ class CliBuildTest < CliTestCase end end - test "verify local dependencies" do - Mrsk::Commands::Builder.any_instance.stubs(:name).returns("remote".inquiry) - - run_command("verify_local_dependencies").tap do |output| - assert_match /docker --version && docker buildx version/, output - end - end - - test "verify local dependencies with no buildx plugin" do - SSHKit::Backend::Abstract.any_instance.stubs(:execute) - .with(:docker, "--version", "&&", :docker, :buildx, "version") - .raises(SSHKit::Command::Failed.new("no buildx")) - - Mrsk::Commands::Builder.any_instance.stubs(:native_and_local?).returns(false) - assert_raises(Mrsk::Cli::Build::BuildError) { run_command("verify_local_dependencies") } - end - private def run_command(*command) stdouted { Mrsk::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) } end - def stub_locking + def stub_dependency_checks SSHKit::Backend::Abstract.any_instance.stubs(:execute) - .with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock } + .with(:docker, "--version", "&&", :docker, :buildx, "version") SSHKit::Backend::Abstract.any_instance.stubs(:execute) - .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" } + .with { |*args| args[0..1] == [:docker, :buildx] } end end diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb index f582cc75..2455002e 100644 --- a/test/cli/cli_test_case.rb +++ b/test/cli/cli_test_case.rb @@ -14,4 +14,32 @@ class CliTestCase < ActiveSupport::TestCase ENV.delete("MYSQL_ROOT_PASSWORD") ENV.delete("VERSION") end + + private + def fail_hook(hook) + @executions = [] + Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) + + SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with { |*args| @executions << args; args != [".mrsk/hooks/#{hook}"] } + SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with { |*args| args.first == ".mrsk/hooks/#{hook}" } + .raises(SSHKit::Command::Failed.new("failed")) + end + + def ensure_hook_runs(hook) + Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) + SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with { |*args| args != [".mrsk/hooks/#{hook}"] } + SSHKit::Backend::Abstract.any_instance.expects(:execute) + .with { |*args| args.first == ".mrsk/hooks/#{hook}" } + .once + end + + def stub_locking + SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with { |arg1, arg2| arg1 == :mkdir && arg2 == :mrsk_lock } + SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with { |arg1, arg2| arg1 == :rm && arg2 == "mrsk_lock/details" } + end end diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index cde31923..b2884b4e 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -10,7 +10,7 @@ class CliMainTest < CliTestCase end test "deploy" do - invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" } + invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options) @@ -20,6 +20,8 @@ class CliMainTest < CliTestCase Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options) + Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) + run_command("deploy").tap do |output| assert_match /Log into image registry/, output assert_match /Build and push app image/, output @@ -27,11 +29,12 @@ class CliMainTest < CliTestCase assert_match /Ensure app can pass healthcheck/, output assert_match /Detect stale containers/, output assert_match /Prune old containers and images/, output + assert_match /Running the post-deploy hook.../, output end end test "deploy with skip_push" do - invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" } + invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options) @@ -81,7 +84,7 @@ class CliMainTest < CliTestCase end test "deploy errors during outside section leave remove lock" do - invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" } + invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } Mrsk::Cli::Main.any_instance.expects(:invoke) .with("mrsk:cli:registry:login", [], invoke_options) @@ -94,22 +97,41 @@ class CliMainTest < CliTestCase assert !MRSK.holding_lock? end + test "deploy with skipped hooks" do + invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => true } + + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:registry:login", [], invoke_options) + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options) + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:traefik:boot", [], invoke_options) + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options) + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options) + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options) + Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:prune:all", [], invoke_options) + + run_command("deploy", "--skip_hooks") do + refute_match /Running the post-deploy hook.../, output + end + end + test "redeploy" do - invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" } + invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:deliver", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:stale_containers", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:app:boot", [], invoke_options) + Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) + run_command("redeploy").tap do |output| assert_match /Build and push app image/, output assert_match /Ensure app can pass healthcheck/, output + assert_match /Running the post-deploy hook.../, output end end test "redeploy with skip_push" do - invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_broadcast" => false, "version" => "999" } + invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:build:pull", [], invoke_options) Mrsk::Cli::Main.any_instance.expects(:invoke).with("mrsk:cli:healthcheck:perform", [], invoke_options) @@ -149,12 +171,14 @@ class CliMainTest < CliTestCase .returns("running").at_least_once # health check end + Mrsk::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output| assert_match "Start container with version 123", output assert_match "docker tag dhh/app:123 dhh/app:latest", output assert_match "docker start app-web-123", output assert_match "docker container ls --all --filter name=^app-web-version-to-rollback$ --quiet | xargs docker stop", output, "Should stop the container that was previously running" + assert_match "Running the post-deploy hook...", output end end @@ -235,9 +259,11 @@ class CliMainTest < CliTestCase end test "init" do - Pathname.any_instance.expects(:exist?).returns(false).twice + Pathname.any_instance.expects(:exist?).returns(false).times(3) + Pathname.any_instance.stubs(:mkpath) FileUtils.stubs(:mkdir_p) FileUtils.stubs(:cp_r) + FileUtils.stubs(:cp) run_command("init").tap do |output| assert_match /Created configuration file in config\/deploy.yml/, output @@ -246,7 +272,7 @@ class CliMainTest < CliTestCase end test "init with existing config" do - Pathname.any_instance.expects(:exist?).returns(true).twice + Pathname.any_instance.expects(:exist?).returns(true).times(3) run_command("init").tap do |output| assert_match /Config file already exists in config\/deploy.yml \(remove first to create a new one\)/, output @@ -254,9 +280,11 @@ class CliMainTest < CliTestCase end test "init with bundle option" do - Pathname.any_instance.expects(:exist?).returns(false).times(3) + Pathname.any_instance.expects(:exist?).returns(false).times(4) + Pathname.any_instance.stubs(:mkpath) FileUtils.stubs(:mkdir_p) FileUtils.stubs(:cp_r) + FileUtils.stubs(:cp) run_command("init", "--bundle").tap do |output| assert_match /Created configuration file in config\/deploy.yml/, output @@ -269,9 +297,11 @@ class CliMainTest < CliTestCase end test "init with bundle option and existing binstub" do - Pathname.any_instance.expects(:exist?).returns(true).times(3) + Pathname.any_instance.expects(:exist?).returns(true).times(4) + Pathname.any_instance.stubs(:mkpath) FileUtils.stubs(:mkdir_p) FileUtils.stubs(:cp_r) + FileUtils.stubs(:cp) run_command("init", "--bundle").tap do |output| assert_match /Config file already exists in config\/deploy.yml \(remove first to create a new one\)/, output @@ -317,19 +347,6 @@ class CliMainTest < CliTestCase end end - test "broadcast" do - SSHKit::Backend::Abstract.any_instance.expects(:execute).with do |command, line, options, verbosity:| - command == "bin/audit_broadcast" && - line =~ /\A'\[[^\]]+\] message'\z/ && - options[:env].keys == %w[ MRSK_RECORDED_AT MRSK_PERFORMER MRSK_EVENT ] && - verbosity == :debug - end.returns("Broadcast audit message: message") - - run_command("broadcast", "-m", "message").tap do |output| - assert_match "Broadcast: message", output - end - end - test "version" do version = stdouted { Mrsk::Cli::Main.new.version } assert_equal Mrsk::VERSION, version diff --git a/test/commands/auditor_test.rb b/test/commands/auditor_test.rb index bcf62929..1a1b03e5 100644 --- a/test/commands/auditor_test.rb +++ b/test/commands/auditor_test.rb @@ -1,19 +1,25 @@ require "test_helper" +require "active_support/testing/time_helpers" class CommandsAuditorTest < ActiveSupport::TestCase + include ActiveSupport::Testing::TimeHelpers + setup do + freeze_time + @config = { - service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], - audit_broadcast_cmd: "bin/audit_broadcast" + service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ] } @auditor = new_command + @performer = `whoami`.strip + @recorded_at = Time.now.utc.iso8601 end test "record" do assert_equal [ :echo, - "[#{@auditor.details[:recorded_at]}]", "[#{@auditor.details[:performer]}]", + "[#{@recorded_at}] [#{@performer}]", "app removed container", ">>", "mrsk-app-audit.log" ], @auditor.record("app removed container") @@ -23,7 +29,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase new_command(destination: "staging").tap do |auditor| assert_equal [ :echo, - "[#{auditor.details[:recorded_at]}]", "[#{auditor.details[:performer]}]", "[#{auditor.details[:destination]}]", + "[#{@recorded_at}] [#{@performer}] [staging]", "app removed container", ">>", "mrsk-app-staging-audit.log" ], auditor.record("app removed container") @@ -34,7 +40,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase new_command(role: "web").tap do |auditor| assert_equal [ :echo, - "[#{auditor.details[:recorded_at]}]", "[#{auditor.details[:performer]}]", "[#{auditor.details[:role]}]", + "[#{@recorded_at}] [#{@performer}] [web]", "app removed container", ">>", "mrsk-app-audit.log" ], auditor.record("app removed container") @@ -44,24 +50,12 @@ class CommandsAuditorTest < ActiveSupport::TestCase test "record with arg details" do assert_equal [ :echo, - "[#{@auditor.details[:recorded_at]}]", "[#{@auditor.details[:performer]}]", "[value]", + "[#{@recorded_at}] [#{@performer}] [value]", "app removed container", ">>", "mrsk-app-audit.log" ], @auditor.record("app removed container", detail: "value") end - test "broadcast" do - assert_equal [ - "bin/audit_broadcast", - "'[#{@auditor.details[:performer]}] [value] app removed container'", - env: { - "MRSK_RECORDED_AT" => @auditor.details[:recorded_at], - "MRSK_PERFORMER" => @auditor.details[:performer], - "MRSK_EVENT" => "app removed container", - "MRSK_DETAIL" => "value" - } - ], @auditor.broadcast("app removed container", detail: "value") - end private def new_command(destination: nil, **details) diff --git a/test/commands/hook_test.rb b/test/commands/hook_test.rb new file mode 100644 index 00000000..726ef7d8 --- /dev/null +++ b/test/commands/hook_test.rb @@ -0,0 +1,44 @@ +require "test_helper" + +class CommandsHookTest < ActiveSupport::TestCase + include ActiveSupport::Testing::TimeHelpers + + setup do + freeze_time + + @config = { + service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], + traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } } + } + + @performer = `whoami`.strip + @recorded_at = Time.now.utc.iso8601 + end + + test "run" do + assert_equal [ + ".mrsk/hooks/foo", + { env: { + "MRSK_RECORDED_AT" => @recorded_at, + "MRSK_PERFORMER" => @performer, + "MRSK_VERSION" => "123", + "MRSK_SERVICE_VERSION" => "app@123" } } + ], new_command.run("foo") + end + + test "run with custom hooks_path" do + assert_equal [ + "custom/hooks/path/foo", + { env: { + "MRSK_RECORDED_AT" => @recorded_at, + "MRSK_PERFORMER" => @performer, + "MRSK_VERSION" => "123", + "MRSK_SERVICE_VERSION" => "app@123" } } + ], new_command(hooks_path: "custom/hooks/path").run("foo") + end + + private + def new_command(**extra_config) + Mrsk::Commands::Hook.new(Mrsk::Configuration.new(@config.merge(**extra_config), version: "123")) + end +end diff --git a/test/fixtures/deploy_simple.yml b/test/fixtures/deploy_simple.yml index 6c3a34ba..520c138e 100644 --- a/test/fixtures/deploy_simple.yml +++ b/test/fixtures/deploy_simple.yml @@ -6,4 +6,3 @@ servers: registry: username: user password: pw -audit_broadcast_cmd: "bin/audit_broadcast" diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml index a3b2db66..0a92d427 100644 --- a/test/integration/docker-compose.yml +++ b/test/integration/docker-compose.yml @@ -17,6 +17,8 @@ services: privileged: true build: context: docker/deployer + environment: + - TEST_ID=${TEST_ID} volumes: - ../..:/mrsk - shared:/shared diff --git a/test/integration/docker/deployer/app/.mrsk/hooks/post-deploy b/test/integration/docker/deployer/app/.mrsk/hooks/post-deploy new file mode 100755 index 00000000..0fcc6920 --- /dev/null +++ b/test/integration/docker/deployer/app/.mrsk/hooks/post-deploy @@ -0,0 +1,3 @@ +#!/bin/sh +echo "Deployed!" +mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/post-deploy diff --git a/test/integration/docker/deployer/app/.mrsk/hooks/pre-build b/test/integration/docker/deployer/app/.mrsk/hooks/pre-build new file mode 100755 index 00000000..0cf78d88 --- /dev/null +++ b/test/integration/docker/deployer/app/.mrsk/hooks/pre-build @@ -0,0 +1,3 @@ +#!/bin/sh +echo "About to build and push..." +mkdir -p /tmp/${TEST_ID} && touch /tmp/${TEST_ID}/pre-build diff --git a/test/integration/docker/deployer/boot.sh b/test/integration/docker/deployer/boot.sh index b25f3334..77d6d1ea 100755 --- a/test/integration/docker/deployer/boot.sh +++ b/test/integration/docker/deployer/boot.sh @@ -1,5 +1,5 @@ #!/bin/bash -dockerd & +dockerd --max-concurrent-downloads 1 & exec sleep infinity diff --git a/test/integration/docker/vm/boot.sh b/test/integration/docker/vm/boot.sh index 81df5cdb..681a8a4e 100755 --- a/test/integration/docker/vm/boot.sh +++ b/test/integration/docker/vm/boot.sh @@ -4,6 +4,6 @@ while [ ! -f /root/.ssh/authorized_keys ]; do echo "Waiting for ssh keys"; sleep service ssh restart -dockerd & +dockerd --max-concurrent-downloads 1 & exec sleep infinity diff --git a/test/integration/integration_test.rb b/test/integration/integration_test.rb index eaf2252b..a35ae775 100644 --- a/test/integration/integration_test.rb +++ b/test/integration/integration_test.rb @@ -3,6 +3,7 @@ require "test_helper" class IntegrationTest < ActiveSupport::TestCase setup do + ENV["TEST_ID"] = SecureRandom.hex docker_compose "up --build -d" wait_for_healthy setup_deployer @@ -14,7 +15,7 @@ class IntegrationTest < ActiveSupport::TestCase private def docker_compose(*commands, capture: false, raise_on_error: true) - command = "docker compose #{commands.join(" ")}" + command = "TEST_ID=#{ENV["TEST_ID"]} docker compose #{commands.join(" ")}" succeeded = false if capture result = stdouted { succeeded = system("cd test/integration && #{command}") } @@ -82,6 +83,25 @@ class IntegrationTest < ActiveSupport::TestCase assert_equal version, response.body.strip end + def assert_hooks_ran(*hooks) + hooks.each do |hook| + file = "/tmp/#{ENV["TEST_ID"]}/#{hook}" + assert_equal "removed '#{file}'", deployer_exec("rm -v #{file}", capture: true).strip + end + end + + def assert_200(response) + code = response.code + if code != "200" + puts "Got response code #{code}, here are the traefik logs:" + mrsk :traefik, :logs + puts "And here are the load balancer logs" + docker_compose :logs, :load_balancer + puts "Tried to get the response code again and got #{app_response.code}" + end + assert_equal "200", code + end + def wait_for_healthy(timeout: 20) timeout_at = Time.now + timeout while docker_compose("ps -a | tail -n +2 | grep -v '(healthy)' | wc -l", capture: true) != "0" diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb index bd4c2dcc..9fecf6b1 100644 --- a/test/integration/main_test.rb +++ b/test/integration/main_test.rb @@ -7,21 +7,20 @@ class MainTest < IntegrationTest assert_app_is_down mrsk :deploy - assert_app_is_up version: first_version + assert_hooks_ran "pre-build", "post-deploy" second_version = update_app_rev mrsk :redeploy - assert_app_is_up version: second_version + assert_hooks_ran "pre-build", "post-deploy" mrsk :rollback, first_version - + assert_hooks_ran "post-deploy" assert_app_is_up version: first_version details = mrsk :details, capture: true - assert_match /Traefik Host: vm1/, details assert_match /Traefik Host: vm2/, details assert_match /App Host: vm1/, details @@ -30,7 +29,6 @@ class MainTest < IntegrationTest assert_match /registry:4443\/app:#{first_version}/, details audit = mrsk :audit, capture: true - assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit end