diff --git a/Gemfile.lock b/Gemfile.lock index 68d79e30..e0d6f61e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -35,13 +35,18 @@ GEM builder (3.2.4) concurrent-ruby (1.2.2) crass (1.0.6) - debug (1.7.1) + debug (1.7.2) + irb (>= 1.5.0) + reline (>= 0.3.1) dotenv (2.8.1) ed25519 (1.3.0) erubi (1.12.0) i18n (1.12.0) concurrent-ruby (~> 1.0) - loofah (2.19.1) + io-console (0.6.0) + irb (1.6.3) + reline (>= 0.3.0) + loofah (2.20.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) method_source (1.0.0) @@ -74,6 +79,8 @@ GEM thor (~> 1.0) zeitwerk (~> 2.5) rake (13.0.6) + reline (0.3.3) + io-console (~> 0.5) ruby2_keywords (0.0.5) sshkit (1.21.4) net-scp (>= 1.1.2) diff --git a/README.md b/README.md index b5936482..8b1f234d 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ Watch the screencast: https://www.youtube.com/watch?v=LL1cV2FXZ5I Join us on Discord: https://discord.gg/YgHVT7GCXS +Ask questions: https://github.com/mrsked/mrsk/discussions + ## Installation If you have a Ruby environment available, you can install MRSK globally with: @@ -14,13 +16,13 @@ If you have a Ruby environment available, you can install MRSK globally with: gem install mrsk ``` -...otherwise, you can run a dockerized version via an alias (add this to your ${SHELL}rc to simplify re-use): +...otherwise, you can run a dockerized version via an alias (add this to your .bashrc or similar to simplify re-use): ```sh alias mrsk='docker run --rm -it -v $HOME/.ssh:/root/.ssh -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}/:/workdir ghcr.io/mrsked/mrsk' ``` -Then, inside your app directory, run `mrsk init` (or `mrsk init --bundle` within Rails apps where you want a bin/mrsk binstub). Now edit the new file `config/deploy.yml`. It could look as simple as this: +Then, inside your app directory, run `mrsk init` (or `mrsk init --bundle` within Rails 7+ apps where you want a bin/mrsk binstub). Now edit the new file `config/deploy.yml`. It could look as simple as this: ```yaml service: hey @@ -191,6 +193,15 @@ ssh: user: app ``` +If you are using non-root user, you need to bootstrap your servers manually, before using them with MRSK. On Ubuntu, you'd do: + +```bash +sudo apt update +sudo apt upgrade -y +sudo apt install -y docker.io curl git +sudo usermod -a -G docker ubuntu +``` + ### Using a proxy SSH host If you need to connect to server through a proxy host, you can use `ssh/proxy`: @@ -476,9 +487,9 @@ We allow users to pass additional docker options to the trafik container like traefik: options: publish: - - 8080:8080 + - 8080:8080 volumes: - - /tmp/example.json:/tmp/example.json + - /tmp/example.json:/tmp/example.json memory: 512m ``` diff --git a/lib/mrsk/cli/main.rb b/lib/mrsk/cli/main.rb index a4743220..83f71c34 100644 --- a/lib/mrsk/cli/main.rb +++ b/lib/mrsk/cli/main.rb @@ -119,7 +119,7 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base desc "config", "Show combined config (including secrets!)" def config run_locally do - puts MRSK.config.to_h.to_yaml + puts Mrsk::Utils.redacted(MRSK.config.to_h).to_yaml end end diff --git a/lib/mrsk/commands/base.rb b/lib/mrsk/commands/base.rb index b381a338..25ca0075 100644 --- a/lib/mrsk/commands/base.rb +++ b/lib/mrsk/commands/base.rb @@ -1,6 +1,6 @@ module Mrsk::Commands class Base - delegate :redact, :argumentize, to: Mrsk::Utils + delegate :sensitive, :argumentize, to: Mrsk::Utils attr_accessor :config diff --git a/lib/mrsk/commands/builder/base.rb b/lib/mrsk/commands/builder/base.rb index 448ecca4..92244510 100644 --- a/lib/mrsk/commands/builder/base.rb +++ b/lib/mrsk/commands/builder/base.rb @@ -28,7 +28,7 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base end def build_args - argumentize "--build-arg", args, redacted: true + argumentize "--build-arg", args, sensitive: true end def build_secrets diff --git a/lib/mrsk/commands/registry.rb b/lib/mrsk/commands/registry.rb index 1276da09..11172eaa 100644 --- a/lib/mrsk/commands/registry.rb +++ b/lib/mrsk/commands/registry.rb @@ -2,7 +2,7 @@ class Mrsk::Commands::Registry < Mrsk::Commands::Base delegate :registry, to: :config def login - docker :login, registry["server"], "-u", redact(lookup("username")), "-p", redact(lookup("password")) + docker :login, registry["server"], "-u", sensitive(lookup("username")), "-p", sensitive(lookup("password")) end def logout diff --git a/lib/mrsk/utils.rb b/lib/mrsk/utils.rb index 08c0c0fa..aab0b18a 100644 --- a/lib/mrsk/utils.rb +++ b/lib/mrsk/utils.rb @@ -2,11 +2,12 @@ module Mrsk::Utils extend self # Return a list of escaped shell arguments using the same named argument against the passed attributes (hash or array). - def argumentize(argument, attributes, redacted: false) + def argumentize(argument, attributes, sensitive: false) Array(attributes).flat_map do |key, value| if value.present? - escaped_pair = [ key, escape_shell_value(value) ].join("=") - [ argument, redacted ? redact(escaped_pair) : escaped_pair ] + attr = "#{key}=#{escape_shell_value(value)}" + attr = self.sensitive(attr, redaction: "#{key}=[REDACTED]") if sensitive + [ argument, attr] else [ argument, key ] end @@ -17,7 +18,7 @@ module Mrsk::Utils # but redacts and expands secrets. def argumentize_env_with_secrets(env) if (secrets = env["secret"]).present? - argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, redacted: true) + argumentize("-e", env["clear"]) + argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, sensitive: true) + argumentize("-e", env["clear"]) else argumentize "-e", env.fetch("clear", env) end @@ -39,9 +40,37 @@ module Mrsk::Utils args.flat_map { |key, value| value.try(:map) { |entry| [key, entry] } || [ [ key, value ] ] } end - # Copied from SSHKit::Backend::Abstract#redact to be available inside Commands classes - def redact(arg) # Used in execute_command to hide redact() args a user passes in - arg.to_s.extend(SSHKit::Redaction) # to_s due to our inability to extend Integer, etc + # Marks sensitive values for redaction in logs and human-visible output. + # Pass `redaction:` to change the default `"[REDACTED]"` redaction, e.g. + # `sensitive "#{arg}=#{secret}", redaction: "#{arg}=xxxx" + def sensitive(...) + Mrsk::Utils::Sensitive.new(...) + end + + def redacted(value) + case + when value.respond_to?(:redaction) + value.redaction + when value.respond_to?(:transform_values) + value.transform_values { |value| redacted value } + when value.respond_to?(:map) + value.map { |element| redacted element } + else + value + end + end + + def unredacted(value) + case + when value.respond_to?(:unredacted) + value.unredacted + when value.respond_to?(:transform_values) + value.transform_values { |value| unredacted value } + when value.respond_to?(:map) + value.map { |element| unredacted element } + else + value + end end # Escape a value to make it safe for shell use. diff --git a/lib/mrsk/utils/sensitive.rb b/lib/mrsk/utils/sensitive.rb new file mode 100644 index 00000000..0d29fab4 --- /dev/null +++ b/lib/mrsk/utils/sensitive.rb @@ -0,0 +1,19 @@ +require "active_support/core_ext/module/delegation" + +class Mrsk::Utils::Sensitive + # So SSHKit knows to redact these values. + include SSHKit::Redaction + + attr_reader :unredacted, :redaction + delegate :to_s, to: :unredacted + delegate :inspect, to: :redaction + + def initialize(value, redaction: "[REDACTED]") + @unredacted, @redaction = value, redaction + end + + # Sensitive values won't leak into YAML output. + def encode_with(coder) + coder.represent_scalar nil, redaction + end +end diff --git a/test/configuration/accessory_test.rb b/test/configuration/accessory_test.rb index 205eebc6..35cc5dfe 100644 --- a/test/configuration/accessory_test.rb +++ b/test/configuration/accessory_test.rb @@ -112,8 +112,11 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase test "env args with secret" do ENV["MYSQL_ROOT_PASSWORD"] = "secret123" - assert_equal ["-e", "MYSQL_ROOT_PASSWORD=\"secret123\"", "-e", "MYSQL_ROOT_HOST=\"%\""], @config.accessory(:mysql).env_args - assert @config.accessory(:mysql).env_args[1].is_a?(SSHKit::Redaction) + + @config.accessory(:mysql).env_args.tap do |env_args| + assert_equal ["-e", "MYSQL_ROOT_PASSWORD=\"secret123\"", "-e", "MYSQL_ROOT_HOST=\"%\""], Mrsk::Utils.unredacted(env_args) + assert_equal ["-e", "MYSQL_ROOT_PASSWORD=[REDACTED]", "-e", "MYSQL_ROOT_HOST=\"%\""], Mrsk::Utils.redacted(env_args) + end ensure ENV["MYSQL_ROOT_PASSWORD"] = nil end diff --git a/test/configuration/role_test.rb b/test/configuration/role_test.rb index 0f83d3df..e61ea51a 100644 --- a/test/configuration/role_test.rb +++ b/test/configuration/role_test.rb @@ -97,7 +97,10 @@ class ConfigurationRoleTest < ActiveSupport::TestCase ENV["REDIS_PASSWORD"] = "secret456" ENV["DB_PASSWORD"] = "secret&\"123" - assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "DB_PASSWORD=\"secret&\\\"123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], @config_with_roles.role(:workers).env_args + @config_with_roles.role(:workers).env_args.tap do |env_args| + assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "DB_PASSWORD=\"secret&\\\"123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.unredacted(env_args) + assert_equal ["-e", "REDIS_PASSWORD=[REDACTED]", "-e", "DB_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.redacted(env_args) + end ensure ENV["REDIS_PASSWORD"] = nil ENV["DB_PASSWORD"] = nil @@ -116,7 +119,10 @@ class ConfigurationRoleTest < ActiveSupport::TestCase ENV["DB_PASSWORD"] = "secret123" - assert_equal ["-e", "DB_PASSWORD=\"secret123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], @config_with_roles.role(:workers).env_args + @config_with_roles.role(:workers).env_args.tap do |env_args| + assert_equal ["-e", "DB_PASSWORD=\"secret123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.unredacted(env_args) + assert_equal ["-e", "DB_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.redacted(env_args) + end ensure ENV["DB_PASSWORD"] = nil end @@ -133,7 +139,10 @@ class ConfigurationRoleTest < ActiveSupport::TestCase ENV["REDIS_PASSWORD"] = "secret456" - assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], @config_with_roles.role(:workers).env_args + @config_with_roles.role(:workers).env_args.tap do |env_args| + assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.unredacted(env_args) + assert_equal ["-e", "REDIS_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.redacted(env_args) + end ensure ENV["REDIS_PASSWORD"] = nil end diff --git a/test/configuration_test.rb b/test/configuration_test.rb index b5282379..ec65686d 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -113,12 +113,13 @@ class ConfigurationTest < ActiveSupport::TestCase test "env args with clear and secrets" do ENV["PASSWORD"] = "secret123" + config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({ env: { "clear" => { "PORT" => "3000" }, "secret" => [ "PASSWORD" ] } }) }) - assert_equal [ "-e", "PASSWORD=\"secret123\"", "-e", "PORT=\"3000\"" ], config.env_args - assert config.env_args[1].is_a?(SSHKit::Redaction) + assert_equal [ "-e", "PASSWORD=\"secret123\"", "-e", "PORT=\"3000\"" ], Mrsk::Utils.unredacted(config.env_args) + assert_equal [ "-e", "PASSWORD=[REDACTED]", "-e", "PORT=\"3000\"" ], Mrsk::Utils.redacted(config.env_args) ensure ENV["PASSWORD"] = nil end @@ -133,12 +134,13 @@ class ConfigurationTest < ActiveSupport::TestCase test "env args with only secrets" do ENV["PASSWORD"] = "secret123" + config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({ env: { "secret" => [ "PASSWORD" ] } }) }) - assert_equal [ "-e", "PASSWORD=\"secret123\"" ], config.env_args - assert config.env_args[1].is_a?(SSHKit::Redaction) + assert_equal [ "-e", "PASSWORD=\"secret123\"" ], Mrsk::Utils.unredacted(config.env_args) + assert_equal [ "-e", "PASSWORD=[REDACTED]" ], Mrsk::Utils.redacted(config.env_args) ensure ENV["PASSWORD"] = nil end diff --git a/test/utils_test.rb b/test/utils_test.rb index f2af428b..5138777f 100644 --- a/test/utils_test.rb +++ b/test/utils_test.rb @@ -8,13 +8,16 @@ class UtilsTest < ActiveSupport::TestCase test "argumentize with redacted" do assert_kind_of SSHKit::Redaction, \ - Mrsk::Utils.argumentize("--label", { foo: "bar" }, redacted: true).last + Mrsk::Utils.argumentize("--label", { foo: "bar" }, sensitive: true).last end test "argumentize_env_with_secrets" do ENV.expects(:fetch).with("FOO").returns("secret") - assert_equal [ "-e", "FOO=\"secret\"", "-e", "BAZ=\"qux\"" ], \ - Mrsk::Utils.argumentize_env_with_secrets({ "secret" => [ "FOO" ], "clear" => { BAZ: "qux" } }) + + args = Mrsk::Utils.argumentize_env_with_secrets({ "secret" => [ "FOO" ], "clear" => { BAZ: "qux" } }) + + assert_equal [ "-e", "FOO=[REDACTED]", "-e", "BAZ=\"qux\"" ], Mrsk::Utils.redacted(args) + assert_equal [ "-e", "FOO=\"secret\"", "-e", "BAZ=\"qux\"" ], Mrsk::Utils.unredacted(args) end test "optionize" do @@ -27,9 +30,20 @@ class UtilsTest < ActiveSupport::TestCase Mrsk::Utils.optionize({ foo: "bar", baz: "qux", quux: true }, with: "=") end - test "redact" do - assert_kind_of SSHKit::Redaction, Mrsk::Utils.redact("secret") - assert_equal "secret", Mrsk::Utils.redact("secret") + test "no redaction from #to_s" do + assert_equal "secret", Mrsk::Utils.sensitive("secret").to_s + end + + test "redact from #inspect" do + assert_equal "[REDACTED]".inspect, Mrsk::Utils.sensitive("secret").inspect + end + + test "redact from SSHKit output" do + assert_kind_of SSHKit::Redaction, Mrsk::Utils.sensitive("secret") + end + + test "redact from YAML output" do + assert_equal "--- ! '[REDACTED]'\n", YAML.dump(Mrsk::Utils.sensitive("secret")) end test "escape_shell_value" do