Merge branch 'main' into pr/176

* main: Simpler Make it explicit, focus on Ubuntu More explicit Not that --bundle is a Rails 7+ option Update README.md Update README.md Add github discussions link to readme Bump debug to fix missing deps in CI Only redact the non-sensitive bits of build args and env vars. improve code sample (traefik configuration)
2023-04-10 14:31:43 +02:00
parent 12632aa7f9 7559f439e9
commit 0eb7a8d087
12 changed files with 126 additions and 32 deletions
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -35,13 +35,18 @@ GEM
    builder (3.2.4)
    concurrent-ruby (1.2.2)
    crass (1.0.6)
-    debug (1.7.1)
+    debug (1.7.2)
+      irb (>= 1.5.0)
+      reline (>= 0.3.1)
    dotenv (2.8.1)
    ed25519 (1.3.0)
    erubi (1.12.0)
    i18n (1.12.0)
      concurrent-ruby (~> 1.0)
-    loofah (2.19.1)
+    io-console (0.6.0)
+    irb (1.6.3)
+      reline (>= 0.3.0)
+    loofah (2.20.0)
      crass (~> 1.0.2)
      nokogiri (>= 1.5.9)
    method_source (1.0.0)
@@ -74,6 +79,8 @@ GEM
      thor (~> 1.0)
      zeitwerk (~> 2.5)
    rake (13.0.6)
+    reline (0.3.3)
+      io-console (~> 0.5)
    ruby2_keywords (0.0.5)
    sshkit (1.21.4)
      net-scp (>= 1.1.2)
--- a/README.md
+++ b/README.md
@@ -6,6 +6,8 @@ Watch the screencast: https://www.youtube.com/watch?v=LL1cV2FXZ5I

 Join us on Discord: https://discord.gg/YgHVT7GCXS

+Ask questions: https://github.com/mrsked/mrsk/discussions
+
 ## Installation

 If you have a Ruby environment available, you can install MRSK globally with:
@@ -14,13 +16,13 @@ If you have a Ruby environment available, you can install MRSK globally with:
 gem install mrsk
 ```

-...otherwise, you can run a dockerized version via an alias (add this to your ${SHELL}rc to simplify re-use):
+...otherwise, you can run a dockerized version via an alias (add this to your .bashrc or similar to simplify re-use):

 ```sh
 alias mrsk='docker run --rm -it -v $HOME/.ssh:/root/.ssh -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}/:/workdir  ghcr.io/mrsked/mrsk'
 ```

-Then, inside your app directory, run `mrsk init` (or `mrsk init --bundle` within Rails apps where you want a bin/mrsk binstub). Now edit the new file `config/deploy.yml`. It could look as simple as this:
+Then, inside your app directory, run `mrsk init` (or `mrsk init --bundle` within Rails 7+ apps where you want a bin/mrsk binstub). Now edit the new file `config/deploy.yml`. It could look as simple as this:

 ```yaml
 service: hey
@@ -191,6 +193,15 @@ ssh:
  user: app
 ```

+If you are using non-root user, you need to bootstrap your servers manually, before using them with MRSK. On Ubuntu, you'd do:
+
+```bash
+sudo apt update
+sudo apt upgrade -y
+sudo apt install -y docker.io curl git
+sudo usermod -a -G docker ubuntu
+```
+
 ### Using a proxy SSH host

 If you need to connect to server through a proxy host, you can use `ssh/proxy`:
@@ -476,9 +487,9 @@ We allow users to pass additional docker options to the trafik container like
 traefik:
  options:
    publish:
-    - 8080:8080
+      - 8080:8080
    volumes:
-    - /tmp/example.json:/tmp/example.json
+      - /tmp/example.json:/tmp/example.json
    memory: 512m
 ```

--- a/lib/mrsk/cli/main.rb
+++ b/lib/mrsk/cli/main.rb
@@ -119,7 +119,7 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
  desc "config", "Show combined config (including secrets!)"
  def config
    run_locally do
-      puts MRSK.config.to_h.to_yaml
+      puts Mrsk::Utils.redacted(MRSK.config.to_h).to_yaml
    end
  end

--- a/lib/mrsk/commands/base.rb
+++ b/lib/mrsk/commands/base.rb
@@ -1,6 +1,6 @@
 module Mrsk::Commands
  class Base
-    delegate :redact, :argumentize, to: Mrsk::Utils
+    delegate :sensitive, :argumentize, to: Mrsk::Utils

    attr_accessor :config

--- a/lib/mrsk/commands/builder/base.rb
+++ b/lib/mrsk/commands/builder/base.rb
@@ -28,7 +28,7 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
    end

    def build_args
-      argumentize "--build-arg", args, redacted: true
+      argumentize "--build-arg", args, sensitive: true
    end

    def build_secrets
--- a/lib/mrsk/commands/registry.rb
+++ b/lib/mrsk/commands/registry.rb
@@ -2,7 +2,7 @@ class Mrsk::Commands::Registry < Mrsk::Commands::Base
  delegate :registry, to: :config

  def login
-    docker :login, registry["server"], "-u", redact(lookup("username")), "-p", redact(lookup("password"))
+    docker :login, registry["server"], "-u", sensitive(lookup("username")), "-p", sensitive(lookup("password"))
  end

  def logout
--- a/lib/mrsk/utils.rb
+++ b/lib/mrsk/utils.rb
@@ -2,11 +2,12 @@ module Mrsk::Utils
  extend self

  # Return a list of escaped shell arguments using the same named argument against the passed attributes (hash or array).
-  def argumentize(argument, attributes, redacted: false)
+  def argumentize(argument, attributes, sensitive: false)
    Array(attributes).flat_map do |key, value|
      if value.present?
-        escaped_pair = [ key, escape_shell_value(value) ].join("=")
-        [ argument, redacted ? redact(escaped_pair) : escaped_pair ]
+        attr = "#{key}=#{escape_shell_value(value)}"
+        attr = self.sensitive(attr, redaction: "#{key}=[REDACTED]") if sensitive
+        [ argument, attr]
      else
        [ argument, key ]
      end
@@ -17,7 +18,7 @@ module Mrsk::Utils
  # but redacts and expands secrets.
  def argumentize_env_with_secrets(env)
    if (secrets = env["secret"]).present?
-      argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, redacted: true) + argumentize("-e", env["clear"])
+      argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, sensitive: true) + argumentize("-e", env["clear"])
    else
      argumentize "-e", env.fetch("clear", env)
    end
@@ -39,9 +40,37 @@ module Mrsk::Utils
    args.flat_map { |key, value| value.try(:map) { |entry| [key, entry] } || [ [ key, value ] ] }
  end

-  # Copied from SSHKit::Backend::Abstract#redact to be available inside Commands classes
-  def redact(arg) # Used in execute_command to hide redact() args a user passes in
-    arg.to_s.extend(SSHKit::Redaction) # to_s due to our inability to extend Integer, etc
+  # Marks sensitive values for redaction in logs and human-visible output.
+  # Pass `redaction:` to change the default `"[REDACTED]"` redaction, e.g.
+  # `sensitive "#{arg}=#{secret}", redaction: "#{arg}=xxxx"
+  def sensitive(...)
+    Mrsk::Utils::Sensitive.new(...)
+  end
+
+  def redacted(value)
+    case
+    when value.respond_to?(:redaction)
+      value.redaction
+    when value.respond_to?(:transform_values)
+      value.transform_values { |value| redacted value }
+    when value.respond_to?(:map)
+      value.map { |element| redacted element }
+    else
+      value
+    end
+  end
+
+  def unredacted(value)
+    case
+    when value.respond_to?(:unredacted)
+      value.unredacted
+    when value.respond_to?(:transform_values)
+      value.transform_values { |value| unredacted value }
+    when value.respond_to?(:map)
+      value.map { |element| unredacted element }
+    else
+      value
+    end
  end

  # Escape a value to make it safe for shell use.
--- a/lib/mrsk/utils/sensitive.rb
+++ b/lib/mrsk/utils/sensitive.rb
@@ -0,0 +1,19 @@
+require "active_support/core_ext/module/delegation"
+
+class Mrsk::Utils::Sensitive
+  # So SSHKit knows to redact these values.
+  include SSHKit::Redaction
+
+  attr_reader :unredacted, :redaction
+  delegate :to_s, to: :unredacted
+  delegate :inspect, to: :redaction
+
+  def initialize(value, redaction: "[REDACTED]")
+    @unredacted, @redaction = value, redaction
+  end
+
+  # Sensitive values won't leak into YAML output.
+  def encode_with(coder)
+    coder.represent_scalar nil, redaction
+  end
+end
--- a/test/configuration/accessory_test.rb
+++ b/test/configuration/accessory_test.rb
@@ -112,8 +112,11 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase

  test "env args with secret" do
    ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
-    assert_equal ["-e", "MYSQL_ROOT_PASSWORD=\"secret123\"", "-e", "MYSQL_ROOT_HOST=\"%\""], @config.accessory(:mysql).env_args
-    assert @config.accessory(:mysql).env_args[1].is_a?(SSHKit::Redaction)
+
+    @config.accessory(:mysql).env_args.tap do |env_args|
+      assert_equal ["-e", "MYSQL_ROOT_PASSWORD=\"secret123\"", "-e", "MYSQL_ROOT_HOST=\"%\""], Mrsk::Utils.unredacted(env_args)
+      assert_equal ["-e", "MYSQL_ROOT_PASSWORD=[REDACTED]", "-e", "MYSQL_ROOT_HOST=\"%\""], Mrsk::Utils.redacted(env_args)
+    end
  ensure
    ENV["MYSQL_ROOT_PASSWORD"] = nil
  end
--- a/test/configuration/role_test.rb
+++ b/test/configuration/role_test.rb
@@ -97,7 +97,10 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
    ENV["REDIS_PASSWORD"] = "secret456"
    ENV["DB_PASSWORD"] = "secret&\"123"

-    assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "DB_PASSWORD=\"secret&\\\"123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], @config_with_roles.role(:workers).env_args
+    @config_with_roles.role(:workers).env_args.tap do |env_args|
+      assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "DB_PASSWORD=\"secret&\\\"123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.unredacted(env_args)
+      assert_equal ["-e", "REDIS_PASSWORD=[REDACTED]", "-e", "DB_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.redacted(env_args)
+    end
  ensure
    ENV["REDIS_PASSWORD"] = nil
    ENV["DB_PASSWORD"] = nil
@@ -116,7 +119,10 @@ class ConfigurationRoleTest < ActiveSupport::TestCase

    ENV["DB_PASSWORD"] = "secret123"

-    assert_equal ["-e", "DB_PASSWORD=\"secret123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], @config_with_roles.role(:workers).env_args
+    @config_with_roles.role(:workers).env_args.tap do |env_args|
+      assert_equal ["-e", "DB_PASSWORD=\"secret123\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.unredacted(env_args)
+      assert_equal ["-e", "DB_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.redacted(env_args)
+    end
  ensure
    ENV["DB_PASSWORD"] = nil
  end
@@ -133,7 +139,10 @@ class ConfigurationRoleTest < ActiveSupport::TestCase

    ENV["REDIS_PASSWORD"] = "secret456"

-    assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], @config_with_roles.role(:workers).env_args
+    @config_with_roles.role(:workers).env_args.tap do |env_args|
+      assert_equal ["-e", "REDIS_PASSWORD=\"secret456\"", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.unredacted(env_args)
+      assert_equal ["-e", "REDIS_PASSWORD=[REDACTED]", "-e", "REDIS_URL=\"redis://a/b\"", "-e", "WEB_CONCURRENCY=\"4\""], Mrsk::Utils.redacted(env_args)
+    end
  ensure
    ENV["REDIS_PASSWORD"] = nil
  end
--- a/test/configuration_test.rb
+++ b/test/configuration_test.rb
@@ -113,12 +113,13 @@ class ConfigurationTest < ActiveSupport::TestCase

  test "env args with clear and secrets" do
    ENV["PASSWORD"] = "secret123"
+
    config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
      env: { "clear" => { "PORT" => "3000" }, "secret" => [ "PASSWORD" ] }
    }) })

-    assert_equal [ "-e", "PASSWORD=\"secret123\"", "-e", "PORT=\"3000\"" ], config.env_args
-    assert config.env_args[1].is_a?(SSHKit::Redaction)
+    assert_equal [ "-e", "PASSWORD=\"secret123\"", "-e", "PORT=\"3000\"" ], Mrsk::Utils.unredacted(config.env_args)
+    assert_equal [ "-e", "PASSWORD=[REDACTED]", "-e", "PORT=\"3000\"" ], Mrsk::Utils.redacted(config.env_args)
  ensure
    ENV["PASSWORD"] = nil
  end
@@ -133,12 +134,13 @@ class ConfigurationTest < ActiveSupport::TestCase

  test "env args with only secrets" do
    ENV["PASSWORD"] = "secret123"
+
    config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
      env: { "secret" => [ "PASSWORD" ] }
    }) })

-    assert_equal [ "-e", "PASSWORD=\"secret123\"" ], config.env_args
-    assert config.env_args[1].is_a?(SSHKit::Redaction)
+    assert_equal [ "-e", "PASSWORD=\"secret123\"" ], Mrsk::Utils.unredacted(config.env_args)
+    assert_equal [ "-e", "PASSWORD=[REDACTED]" ], Mrsk::Utils.redacted(config.env_args)
  ensure
    ENV["PASSWORD"] = nil
  end
--- a/test/utils_test.rb
+++ b/test/utils_test.rb
@@ -8,13 +8,16 @@ class UtilsTest < ActiveSupport::TestCase

  test "argumentize with redacted" do
    assert_kind_of SSHKit::Redaction, \
-      Mrsk::Utils.argumentize("--label", { foo: "bar" }, redacted: true).last
+      Mrsk::Utils.argumentize("--label", { foo: "bar" }, sensitive: true).last
  end

  test "argumentize_env_with_secrets" do
    ENV.expects(:fetch).with("FOO").returns("secret")
-    assert_equal [ "-e", "FOO=\"secret\"", "-e", "BAZ=\"qux\"" ], \
-      Mrsk::Utils.argumentize_env_with_secrets({ "secret" => [ "FOO" ], "clear" => { BAZ: "qux" } })
+
+    args = Mrsk::Utils.argumentize_env_with_secrets({ "secret" => [ "FOO" ], "clear" => { BAZ: "qux" } })
+
+    assert_equal [ "-e", "FOO=[REDACTED]", "-e", "BAZ=\"qux\"" ], Mrsk::Utils.redacted(args)
+    assert_equal [ "-e", "FOO=\"secret\"", "-e", "BAZ=\"qux\"" ], Mrsk::Utils.unredacted(args)
  end

  test "optionize" do
@@ -27,9 +30,20 @@ class UtilsTest < ActiveSupport::TestCase
      Mrsk::Utils.optionize({ foo: "bar", baz: "qux", quux: true }, with: "=")
  end

-  test "redact" do
-    assert_kind_of SSHKit::Redaction, Mrsk::Utils.redact("secret")
-    assert_equal "secret", Mrsk::Utils.redact("secret")
+  test "no redaction from #to_s" do
+    assert_equal "secret", Mrsk::Utils.sensitive("secret").to_s
+  end
+
+  test "redact from #inspect" do
+    assert_equal "[REDACTED]".inspect, Mrsk::Utils.sensitive("secret").inspect
+  end
+
+  test "redact from SSHKit output" do
+    assert_kind_of SSHKit::Redaction, Mrsk::Utils.sensitive("secret")
+  end
+
+  test "redact from YAML output" do
+    assert_equal "--- ! '[REDACTED]'\n", YAML.dump(Mrsk::Utils.sensitive("secret"))
  end

  test "escape_shell_value" do