Bump version for 0.4.0

Only log errors

Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    mrsk (0.3.1)
+    mrsk (0.4.0)
       activesupport (>= 7.0)
+      dotenv (~> 2.8)
       sshkit (~> 1.21)
       thor (~> 1.2)
 
@@ -33,6 +34,7 @@ GEM
     debug (1.7.1)
       irb (>= 1.5.0)
       reline (>= 0.3.1)
+    dotenv (2.8.1)
     erubi (1.12.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)

README.md

@@ -46,6 +46,15 @@ Kubernetes is a beast. Running it yourself on your own hardware is not for the f
 
 ## Configuration
 
+### Using .env file to load required environment variables
+
+MRSK uses [dotenv](https://github.com/bkeepers/dotenv) to automatically load environment variables set in the `.env` file present in the application root. This file can be used to set variables like `MRSK_REGISTRY_PASSWORD` or database passwords. But for this reason you must ensure that .env files are not checked into Git or included in your Dockerfile! The format is just key-value like:
+
+```bash
+MRSK_REGISTRY_PASSWORD=pw
+DB_PASSWORD=secret123
+```
+
 ### Using another registry than Docker Hub
 
 The default registry is Docker Hub, but you can change it using `registry/server`:
@@ -226,6 +235,18 @@ RUN --mount=type=secret,id=GITHUB_TOKEN \
   bundle install
 ```
 
+### Using command arguments for Traefik
+
+You can customize the traefik command line:
+
+```yaml
+traefik:
+  accesslog: true
+  accesslog.format: json
+  metrics.prometheus: true
+  metrics.prometheus.buckets: 0.1,0.3,1.2,5.0
+```
+
 ### Configuring build args for new images
 
 Build arguments that aren't secret can also be configured:

bin/mrsk

@@ -3,6 +3,7 @@
 # Prevent failures from being reported twice.
 Thread.report_on_exception = false
 
+require "dotenv/load"
 require "mrsk/cli"
 
 begin
@@ -10,4 +11,7 @@ begin
 rescue SSHKit::Runner::ExecuteError => e
   puts "  \e[31mERROR (#{e.cause.class}): #{e.cause.message}\e[0m"
   puts e.cause.backtrace if ENV["VERBOSE"]
+rescue => e
+  puts "  \e[31mERROR (#{e.class}): #{e.message}\e[0m"
+  puts e.backtrace if ENV["VERBOSE"]
 end

lib/mrsk/cli/accessory.rb

@@ -83,11 +83,29 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   end
 
   desc "exec [NAME] [CMD]", "Execute a custom command on accessory host"
-  option :run, type: :boolean, default: false, desc: "Start a new container to run the command rather than reusing existing"
+  option :method, aliases: "-m", default: "exec", desc: "Execution method: [exec] perform inside container / [run] perform in new container / [ssh] perform over ssh"
   def exec(name, cmd)
+    runner = \
+      case options[:method]
+      when "exec"     then "exec"
+      when "run"      then "run_exec"
+      when "ssh_exec" then "exec_over_ssh"
+      when "ssh_run"  then "run_over_ssh"
+      else raise "Unknown method: #{options[:method]}"
+      end.inquiry
+
     with_accessory(name) do |accessory|
-      runner = options[:run] ? :run_exec : :exec
-      on(accessory.host) { |host| puts_by_host host, capture_with_info(*accessory.send(runner, cmd)) }
+      if runner.exec_over_ssh? || runner.run_over_ssh?
+        run_locally do
+          info "Launching command on #{accessory.host}"
+          exec accessory.send(runner, cmd)
+        end
+      else
+        on(accessory.host) do
+          info "Launching command on #{accessory.host}"
+          execute *accessory.send(runner, cmd)
+        end
+      end
     end
   end
 
@@ -96,7 +114,7 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
     with_accessory(name) do |accessory|
       run_locally do
         info "Launching bash session on #{accessory.host}"
-        exec accessory.bash(host: accessory.host)
+        exec accessory.bash
       end
     end
   end
@@ -118,7 +136,7 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
         end
       else
         since = options[:since]
-        lines = options[:lines]
+        lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
         on(accessory.host) do
           puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep))

lib/mrsk/cli/app.rb

@@ -109,7 +109,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
       end
     else
       since = options[:since]
-      lines = options[:lines]
+      lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
       on(MRSK.hosts) do |host|
         begin

lib/mrsk/cli/base.rb

@@ -8,6 +8,7 @@ module Mrsk::Cli
     def self.exit_on_failure?() true end
 
     class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging"
+    class_option :quiet, type: :boolean, aliases: "-q", desc: "Minimal logging"
 
     class_option :version, desc: "Run commands against a specific app version"
 
@@ -28,12 +29,20 @@ module Mrsk::Cli
         MRSK.tap do |commander|
           commander.config_file = Pathname.new(File.expand_path(options[:config_file]))
           commander.destination = options[:destination]
-          commander.verbose     = options[:verbose]
           commander.version     = options[:version]
 
           commander.specific_hosts    = options[:hosts]&.split(",")
           commander.specific_roles    = options[:roles]&.split(",")
           commander.specific_primary! if options[:primary]
+
+          if options[:verbose]
+            ENV["VERBOSE"] = "1" # For backtraces via cli/start
+            commander.verbosity = :debug
+          end
+
+          if options[:quiet]
+            commander.verbosity = :error
+          end
         end
       end
 

lib/mrsk/cli/build.rb

@@ -9,18 +9,17 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
 
   desc "push", "Build locally and push app image to registry"
   def push
-    verbose = options[:verbose]
     cli = self
 
     run_locally do 
       begin
-        MRSK.verbosity(:debug) { execute *MRSK.builder.push }
+        MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
       rescue SSHKit::Command::Failed => e
         if e.message =~ /(no builder)|(no such file or directory)/
           error "Missing compatible builder, so creating a new one first"
 
           if cli.create
-            MRSK.verbosity(:debug) { execute *MRSK.builder.push }
+            MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
           end
         else
           raise

lib/mrsk/cli/traefik.rb

@@ -50,7 +50,7 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
       end
     else
       since = options[:since]
-      lines = options[:lines]
+      lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
       on(MRSK.traefik_hosts) do |host|
         puts_by_host host, capture(*MRSK.traefik.logs(since: since, lines: lines, grep: grep)), type: "Traefik"

lib/mrsk/commander.rb

@@ -9,10 +9,10 @@ require "mrsk/commands/traefik"
 require "mrsk/commands/registry"
 
 class Mrsk::Commander
-  attr_accessor :config_file, :destination, :verbose, :version
+  attr_accessor :config_file, :destination, :verbosity, :version
 
-  def initialize(config_file: nil, destination: nil, verbose: false)
-    @config_file, @destination, @verbose = config_file, destination, verbose
+  def initialize(config_file: nil, destination: nil, verbosity: :info)
+    @config_file, @destination, @verbosity = config_file, destination, verbosity
   end
 
   def config
@@ -78,7 +78,7 @@ class Mrsk::Commander
   end
 
 
-  def verbosity(level) 
+  def with_verbosity(level) 
     old_level = SSHKit.config.output_verbosity
     SSHKit.config.output_verbosity = level
     yield
@@ -95,6 +95,6 @@ class Mrsk::Commander
     def configure_sshkit_with(config)
       SSHKit::Backend::Netssh.configure { |ssh| ssh.ssh_options = config.ssh_options }
       SSHKit.config.command_map[:docker] = "docker" # No need to use /usr/bin/env, just clogs up the logs
-      SSHKit.config.output_verbosity = :debug if verbose
+      SSHKit.config.output_verbosity = verbosity
     end
 end

lib/mrsk/commands/accessory.rb

@@ -42,15 +42,13 @@ class Mrsk::Commands::Accessory < Mrsk::Commands::Base
   def follow_logs(grep: nil)
     run_over_ssh pipe(
       docker(:logs, service_name, "-t", "-n", "10", "-f", "2>&1"),
-      ("grep '#{grep}'" if grep)
-    ).join(" "), host: host
+      (%(grep "#{grep}") if grep)
+    ).join(" ")
   end
 
   def exec(*command, interactive: false)
     docker :exec,
       ("-it" if interactive),
-      *env_args,
-      *volume_args,
       service_name,
       *command
   end
@@ -65,8 +63,16 @@ class Mrsk::Commands::Accessory < Mrsk::Commands::Base
       *command
   end
 
-  def bash(host:)
-    exec_over_ssh "bash", host: host
+  def run_over_ssh(command)
+    super command, host: host
+  end
+
+  def exec_over_ssh(*command)
+    run_over_ssh run_exec(*command, interactive: true).join(" ")
+  end
+
+  def bash
+    exec_over_ssh "bash"
   end
 
   def ensure_local_file_present(local_file)
@@ -96,10 +102,6 @@ class Mrsk::Commands::Accessory < Mrsk::Commands::Base
   end
 
   private
-    def exec_over_ssh(*command, host:)
-      run_over_ssh run_exec(*command, interactive: true).join(" "), host: host
-    end
-
     def service_filter
       [ "--filter", "label=service=#{service_name}" ]
     end

lib/mrsk/commands/app.rb

@@ -35,16 +35,13 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   def logs(since: nil, lines: nil, grep: nil)
     pipe \
       current_container_id,
-      "xargs docker logs#{" --since #{since}" if since}#{" -n #{lines}" if lines} -t 2>&1",
+      "xargs docker logs#{" --since #{since}" if since}#{" -n #{lines}" if lines} 2>&1",
       ("grep '#{grep}'" if grep)
   end
 
   def exec(*command, interactive: false)
     docker :exec,
       ("-it" if interactive),
-      *rails_master_key_arg,
-      *config.env_args,
-      *config.volume_args,
       config.service_with_version,
       *command
   end
@@ -68,7 +65,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
     run_over_ssh pipe(
       current_container_id,
       "xargs docker logs -t -n 10 -f 2>&1",
-      ("grep '#{grep}'" if grep)
+      (%(grep "#{grep}") if grep)
     ).join(" "), host: host
   end
 

lib/mrsk/commands/base.rb

@@ -8,6 +8,10 @@ module Mrsk::Commands
       @config = config
     end
 
+    def run_over_ssh(command, host:)
+      "ssh -t #{config.ssh_user}@#{host} '#{command}'"
+    end
+
     private
       def combine(*commands, by: "&&")
         commands
@@ -27,9 +31,5 @@ module Mrsk::Commands
       def docker(*args)
         args.compact.unshift :docker
       end
-
-      def run_over_ssh(command, host:)
-        "ssh -t #{config.ssh_user}@#{host} '#{command}'"
-      end
   end
 end

lib/mrsk/commands/traefik.rb

@@ -9,7 +9,8 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
       "-v /var/run/docker.sock:/var/run/docker.sock",
       "traefik",
       "--providers.docker",
-      "--log.level=DEBUG"
+      "--log.level=DEBUG",
+      *cmd_args
   end
 
   def start
@@ -33,7 +34,7 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
   def follow_logs(host:, grep: nil)
     run_over_ssh pipe(
       docker(:logs, "traefik", "-t", "-n", "10", "-f", "2>&1"),
-      ("grep '#{grep}'" if grep)
+      (%(grep "#{grep}") if grep)
     ).join(" "), host: host
   end
 
@@ -44,4 +45,9 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
   def remove_image
     docker :image, :prune, "-a", "-f", "--filter", "label=org.opencontainers.image.title=Traefik"
   end
+
+  private
+    def cmd_args
+      (config.raw_config.dig(:traefik, "args") || { }).collect { |(key, value)| [ "--#{key}", value ] }.flatten
+    end
 end

lib/mrsk/configuration.rb

@@ -39,7 +39,7 @@ class Mrsk::Configuration
   def initialize(raw_config, version: "missing", validate: true)
     @raw_config = ActiveSupport::InheritableOptions.new(raw_config)
     @version = version
-    ensure_required_keys_present if validate
+    valid? if validate
   end
 
 
@@ -120,6 +120,12 @@ class Mrsk::Configuration
     end
   end
 
+
+  def valid?
+    ensure_required_keys_present && ensure_env_available
+  end
+
+
   def to_h
     {
       roles: role_names,
@@ -139,6 +145,7 @@ class Mrsk::Configuration
 
 
   private
+    # Will raise ArgumentError if any required config keys are missing
     def ensure_required_keys_present
       %i[ service image registry servers ].each do |key|
         raise ArgumentError, "Missing required configuration for #{key}" unless raw_config[key].present?
@@ -151,6 +158,16 @@ class Mrsk::Configuration
       if raw_config.registry["password"].blank?
         raise ArgumentError, "You must specify a password for the registry in config/deploy.yml (or set the ENV variable if that's used)"
       end
+
+      true
+    end
+
+    # Will raise KeyError if any secret ENVs are missing
+    def ensure_env_available
+      env_args
+      roles.each(&:env_args)
+
+      true
     end
 
     def role_names

lib/mrsk/configuration/accessory.rb

@@ -74,12 +74,19 @@ class Mrsk::Configuration::Assessory
 
     def expand_local_file(local_file)
       if local_file.end_with?("erb")
-        read_dynamic_file(local_file)
+        with_clear_env_loaded { read_dynamic_file(local_file) }
       else
         Pathname.new(File.expand_path(local_file)).to_s
       end
     end
 
+    def with_clear_env_loaded
+      (env["clear"] || env).each { |k, v| ENV[k] = v }
+      yield
+    ensure
+      (env["clear"] || env).each { |k, v| ENV.delete(k) }
+    end
+
     def read_dynamic_file(local_file)
       StringIO.new(ERB.new(IO.read(local_file)).result)
     end

lib/mrsk/utils.rb

@@ -18,7 +18,7 @@ module Mrsk::Utils
     if (secrets = env["secret"]).present?
       argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, redacted: true) + argumentize("-e", env["clear"])
     else
-      argumentize "-e", env
+      argumentize "-e", env.fetch("clear", env)
     end
   end
 

lib/mrsk/version.rb

@@ -1,3 +1,3 @@
 module Mrsk
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

mrsk.gemspec

@@ -15,4 +15,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "activesupport", ">= 7.0"
   spec.add_dependency "sshkit", "~> 1.21"
   spec.add_dependency "thor", "~> 1.2"
+  spec.add_dependency "dotenv", "~> 2.8"
 end

test/commands/traefik_test.rb

@@ -0,0 +1,23 @@
+require "test_helper"
+require "mrsk/configuration"
+require "mrsk/commands/traefik"
+
+class CommandsTraefikTest < ActiveSupport::TestCase
+  setup do
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
+    }
+  end
+
+  test "run" do
+    assert_equal \
+      [:docker, :run, "--name traefik", "-d", "--restart unless-stopped", "-p 80:80", "-v /var/run/docker.sock:/var/run/docker.sock", "traefik", "--providers.docker", "--log.level=DEBUG", "--accesslog.format", "json", "--metrics.prometheus.buckets", "0.1,0.3,1.2,5.0"],
+      new_command.run
+  end
+
+  private
+    def new_command
+      Mrsk::Commands::Traefik.new(Mrsk::Configuration.new(@config, version: "123"))
+    end
+end

test/configuration/accessory_test.rb

@@ -98,7 +98,8 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
     @deploy[:accessories]["mysql"]["files"] << "test/fixtures/files/structure.sql.erb:/docker-entrypoint-initdb.d/structure.sql"
     @config = Mrsk::Configuration.new(@deploy)
 
-    assert_equal "This was dynamically expanded", @config.accessory(:mysql).files.keys[2].read
+    assert_match "This was dynamically expanded", @config.accessory(:mysql).files.keys[2].read
+    assert_match "%", @config.accessory(:mysql).files.keys[2].read
   end
 
   test "directories" do

test/configuration_test.rb

@@ -105,6 +105,14 @@ class ConfigurationTest < ActiveSupport::TestCase
     ENV["PASSWORD"] = nil
   end
 
+  test "env args with only clear" do
+    config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
+      env: { "clear" => { "PORT" => "3000" } }
+    }) })
+
+    assert_equal [ "-e", "PORT=3000" ], config.env_args
+  end
+
   test "env args with only secrets" do
     ENV["PASSWORD"] = "secret123"
     config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
@@ -118,15 +126,17 @@ class ConfigurationTest < ActiveSupport::TestCase
   end
 
   test "env args with missing secret" do
-    config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
-      env: { "secret" => [ "PASSWORD" ] }
-    }) })
-
     assert_raises(KeyError) do
-      assert_equal [ "-e", "PASSWORD=secret123" ], config.env_args
+      config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
+        env: { "secret" => [ "PASSWORD" ] }
+      }) })
     end
   end
 
+  test "valid config" do
+    assert @config.valid?
+  end
+
   test "ssh options" do
     assert_equal "root", @config.ssh_options[:user]
 

test/fixtures/files/structure.sql.erb

@@ -1 +1,2 @@
-<%= "This was dynamically expanded" %>
+<%= "This was dynamically expanded" %>
+<%= ENV["MYSQL_ROOT_HOST"] %>