Bump version for 0.4.0

Only log errors

Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    mrsk (0.2.0)
+    mrsk (0.4.0)
       activesupport (>= 7.0)
+      dotenv (~> 2.8)
       sshkit (~> 1.21)
       thor (~> 1.2)
 
@@ -33,6 +34,7 @@ GEM
     debug (1.7.1)
       irb (>= 1.5.0)
       reline (>= 0.3.1)
+    dotenv (2.8.1)
     erubi (1.12.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)

README.md

@@ -46,6 +46,15 @@ Kubernetes is a beast. Running it yourself on your own hardware is not for the f
 
 ## Configuration
 
+### Using .env file to load required environment variables
+
+MRSK uses [dotenv](https://github.com/bkeepers/dotenv) to automatically load environment variables set in the `.env` file present in the application root. This file can be used to set variables like `MRSK_REGISTRY_PASSWORD` or database passwords. But for this reason you must ensure that .env files are not checked into Git or included in your Dockerfile! The format is just key-value like:
+
+```bash
+MRSK_REGISTRY_PASSWORD=pw
+DB_PASSWORD=secret123
+```
+
 ### Using another registry than Docker Hub
 
 The default registry is Docker Hub, but you can change it using `registry/server`:
@@ -226,6 +235,18 @@ RUN --mount=type=secret,id=GITHUB_TOKEN \
   bundle install
 ```
 
+### Using command arguments for Traefik
+
+You can customize the traefik command line:
+
+```yaml
+traefik:
+  accesslog: true
+  accesslog.format: json
+  metrics.prometheus: true
+  metrics.prometheus.buckets: 0.1,0.3,1.2,5.0
+```
+
 ### Configuring build args for new images
 
 Build arguments that aren't secret can also be configured:
@@ -244,6 +265,14 @@ ARG RUBY_VERSION
 FROM ruby:$RUBY_VERSION-slim as base
 ```
 
+### Using without RAILS_MASTER_KEY
+
+If you're using MRSK with older Rails apps that predate RAILS_MASTER_KEY, or with a non-Rails app, you can skip the default usage and reference:
+
+```yaml
+skip_master_key: true
+```
+
 ### Using accessories for database, cache, search services
 
 You can manage your accessory services via MRSK as well. The services will build off public images, and will not be automatically updated when you deploy:

bin/mrsk

@@ -1,5 +1,17 @@
 #!/usr/bin/env ruby
 
+# Prevent failures from being reported twice.
+Thread.report_on_exception = false
+
+require "dotenv/load"
 require "mrsk/cli"
 
-Mrsk::Cli::Main.start(ARGV)
+begin
+  Mrsk::Cli::Main.start(ARGV)
+rescue SSHKit::Runner::ExecuteError => e
+  puts "  \e[31mERROR (#{e.cause.class}): #{e.cause.message}\e[0m"
+  puts e.cause.backtrace if ENV["VERBOSE"]
+rescue => e
+  puts "  \e[31mERROR (#{e.class}): #{e.message}\e[0m"
+  puts e.backtrace if ENV["VERBOSE"]
+end

lib/mrsk/cli/accessory.rb

@@ -1,41 +1,122 @@
 require "mrsk/cli/base"
 
 class Mrsk::Cli::Accessory < Mrsk::Cli::Base
-  desc "boot [NAME]", "Boot accessory service on host"
+  desc "boot [NAME]", "Boot accessory service on host (use NAME=all to boot all accessories)"
   def boot(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.run }
+    if name == "all"
+      MRSK.accessory_names.each { |accessory_name| boot(accessory_name) }
+    else
+      with_accessory(name) do |accessory|
+        directories(name)
+        upload(name)
+        on(accessory.host) { execute *accessory.run }
+      end
+    end
+  end
+
+  desc "upload [NAME]", "Upload accessory files to host"
+  def upload(name)
+    with_accessory(name) do |accessory|
+      on(accessory.host) do
+        accessory.files.each do |(local, remote)|
+          accessory.ensure_local_file_present(local)
+
+          execute *accessory.make_directory_for(remote)
+          upload! local, remote
+          execute :chmod, "755", remote
+        end
+      end
+    end
+  end
+
+  desc "directories [NAME]", "Create accessory directories on host"
+  def directories(name)
+    with_accessory(name) do |accessory|
+      on(accessory.host) do
+        accessory.directories.keys.each do |host_path|
+          execute *accessory.make_directory(host_path)
+        end
+      end
+    end
   end
 
   desc "reboot [NAME]", "Reboot accessory on host (stop container, remove container, start new container)"
   def reboot(name)
-    invoke :stop, [ name ]
-    invoke :remove_container, [ name ]
-    invoke :boot, [ name ]
+    with_accessory(name) do |accessory|
+      stop(name)
+      remove_container(name)
+      boot(name)
+    end
   end
 
   desc "start [NAME]", "Start existing accessory on host"
   def start(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.start }
+    with_accessory(name) do |accessory|
+      on(accessory.host) { execute *accessory.start }
+    end
   end
 
   desc "stop [NAME]", "Stop accessory on host"
   def stop(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.stop }
+    with_accessory(name) do |accessory|
+      on(accessory.host) { execute *accessory.stop, raise_on_non_zero_exit: false }
+    end
   end
 
   desc "restart [NAME]", "Restart accessory on host"
   def restart(name)
-    invoke :stop, [ name ]
-    invoke :start, [ name ]
+    with_accessory(name) do
+      stop(name)
+      start(name)
+    end
   end
 
-  desc "details [NAME]", "Display details about accessory on host"
+  desc "details [NAME]", "Display details about accessory on host (use NAME=all to boot all accessories)"
   def details(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { puts capture_with_info(*accessory.info) }
+    if name == "all"
+      MRSK.accessory_names.each { |accessory_name| details(accessory_name) }
+    else
+      with_accessory(name) do |accessory|
+        on(accessory.host) { puts capture_with_info(*accessory.info) }
+      end
+    end
+  end
+
+  desc "exec [NAME] [CMD]", "Execute a custom command on accessory host"
+  option :method, aliases: "-m", default: "exec", desc: "Execution method: [exec] perform inside container / [run] perform in new container / [ssh] perform over ssh"
+  def exec(name, cmd)
+    runner = \
+      case options[:method]
+      when "exec"     then "exec"
+      when "run"      then "run_exec"
+      when "ssh_exec" then "exec_over_ssh"
+      when "ssh_run"  then "run_over_ssh"
+      else raise "Unknown method: #{options[:method]}"
+      end.inquiry
+
+    with_accessory(name) do |accessory|
+      if runner.exec_over_ssh? || runner.run_over_ssh?
+        run_locally do
+          info "Launching command on #{accessory.host}"
+          exec accessory.send(runner, cmd)
+        end
+      else
+        on(accessory.host) do
+          info "Launching command on #{accessory.host}"
+          execute *accessory.send(runner, cmd)
+        end
+      end
+    end
+  end
+
+  desc "bash [NAME]", "Start a bash session on primary host (or specific host set by --hosts)"
+  def bash(name)
+    with_accessory(name) do |accessory|
+      run_locally do
+        info "Launching bash session on #{accessory.host}"
+        exec accessory.bash
+      end
+    end
   end
 
   desc "logs [NAME]", "Show log lines from accessory on host"
@@ -44,42 +125,75 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
   option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
   def logs(name)
-    accessory = MRSK.accessory(name)
+    with_accessory(name) do |accessory|
+      grep = options[:grep]
 
-    grep = options[:grep]
+      if options[:follow]
+        run_locally do
+          info "Following logs on #{accessory.host}..."
+          info accessory.follow_logs(grep: grep)
+          exec accessory.follow_logs(grep: grep)
+        end
+      else
+        since = options[:since]
+        lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
-    if options[:follow]
-      run_locally do
-        info "Following logs on #{accessory.host}..."
-        info accessory.follow_logs(grep: grep)
-        exec accessory.follow_logs(grep: grep)
-      end
-    else
-      since = options[:since]
-      lines = options[:lines]
-
-      on(accessory.host) do
-        puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep))
+        on(accessory.host) do
+          puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep))
+        end
       end
     end
   end
 
-  desc "remove [NAME]", "Remove accessory container and image from host"
+  desc "remove [NAME]", "Remove accessory container and image from host (use NAME=all to boot all accessories)"
   def remove(name)
-    invoke :stop, [ name ]
-    invoke :remove_container, [ name ]
-    invoke :remove_image, [ name ]
+    if name == "all"
+      MRSK.accessory_names.each { |accessory_name| remove(accessory_name) }
+    else
+      with_accessory(name) do
+        stop(name)
+        remove_container(name)
+        remove_image(name)
+        remove_service_directory(name)
+      end
+    end
   end
 
   desc "remove_container [NAME]", "Remove accessory container from host"
   def remove_container(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.remove_container }
+    with_accessory(name) do |accessory|
+      on(accessory.host) { execute *accessory.remove_container }
+    end
   end
 
-  desc "remove_container [NAME]", "Remove accessory image from servers"
+  desc "remove_container [NAME]", "Remove accessory image from host"
   def remove_image(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.remove_image }
+    with_accessory(name) do |accessory|
+      on(accessory.host) { execute *accessory.remove_image }
+    end
   end
+
+  desc "remove_service_directory [NAME]", "Remove accessory directory used for uploaded files and data directories from host"
+  def remove_service_directory(name)
+    with_accessory(name) do |accessory|
+      on(accessory.host) { execute *accessory.remove_service_directory }
+    end
+  end
+
+  private
+    def with_accessory(name)
+      if accessory = MRSK.accessory(name)
+        yield accessory
+      else
+        error_on_missing_accessory(name)
+      end
+    end
+
+    def error_on_missing_accessory(name)
+      options = MRSK.accessory_names.presence
+
+      error \
+        "No accessory by the name of '#{name}'" +
+        (options ? " (options: #{options.to_sentence})" : "")
+    end
 end

lib/mrsk/cli/app.rb

@@ -40,10 +40,24 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   end
   
   desc "exec [CMD]", "Execute a custom command on servers"
-  option :run, type: :boolean, default: false, desc: "Start a new container to run the command rather than reusing existing"
+  option :method, aliases: "-m", default: "exec", desc: "Execution method: [exec] perform inside app container / [run] perform in new container / [ssh] perform over ssh"
   def exec(cmd)
-    runner = options[:run] ? :run_exec : :exec
-    on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.send(runner, cmd)) }
+    runner = \
+      case options[:method]
+      when "exec" then "exec"
+      when "run"  then "run_exec"
+      when "ssh"  then "exec_over_ssh"
+      else raise "Unknown method: #{options[:method]}"
+      end.inquiry
+
+    if runner.exec_over_ssh?
+      run_locally do
+        info "Launching command on #{MRSK.primary_host}"
+        exec MRSK.app.exec_over_ssh(cmd, host: MRSK.primary_host)
+      end
+    else
+      on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.send(runner, cmd)) }
+    end
   end
 
   desc "console", "Start Rails Console on primary host (or specific host set by --hosts)"
@@ -95,7 +109,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
       end
     else
       since = options[:since]
-      lines = options[:lines]
+      lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
       on(MRSK.hosts) do |host|
         begin

lib/mrsk/cli/base.rb

@@ -8,6 +8,7 @@ module Mrsk::Cli
     def self.exit_on_failure?() true end
 
     class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging"
+    class_option :quiet, type: :boolean, aliases: "-q", desc: "Minimal logging"
 
     class_option :version, desc: "Run commands against a specific app version"
 
@@ -28,12 +29,20 @@ module Mrsk::Cli
         MRSK.tap do |commander|
           commander.config_file = Pathname.new(File.expand_path(options[:config_file]))
           commander.destination = options[:destination]
-          commander.verbose     = options[:verbose]
           commander.version     = options[:version]
 
           commander.specific_hosts    = options[:hosts]&.split(",")
           commander.specific_roles    = options[:roles]&.split(",")
           commander.specific_primary! if options[:primary]
+
+          if options[:verbose]
+            ENV["VERBOSE"] = "1" # For backtraces via cli/start
+            commander.verbosity = :debug
+          end
+
+          if options[:quiet]
+            commander.verbosity = :error
+          end
         end
       end
 

lib/mrsk/cli/build.rb

@@ -9,15 +9,21 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
 
   desc "push", "Build locally and push app image to registry"
   def push
-    verbose = options[:verbose]
+    cli = self
 
     run_locally do 
       begin
-        MRSK.verbosity(:debug) { execute *MRSK.builder.push }
+        MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
       rescue SSHKit::Command::Failed => e
-        error "Missing compatible builder, so creating a new one first"
-        execute *MRSK.builder.create
-        MRSK.verbosity(:debug) { execute *MRSK.builder.push }
+        if e.message =~ /(no builder)|(no such file or directory)/
+          error "Missing compatible builder, so creating a new one first"
+
+          if cli.create
+            MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
+          end
+        else
+          raise
+        end
       end
     end
   end
@@ -30,8 +36,17 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
   desc "create", "Create a local build setup"
   def create
     run_locally do
-      debug "Using builder: #{MRSK.builder.name}"
-      execute *MRSK.builder.create
+      begin
+        debug "Using builder: #{MRSK.builder.name}"
+        execute *MRSK.builder.create
+      rescue SSHKit::Command::Failed => e
+        if e.message =~ /stderr=(.*)/
+          error "Couldn't create remote builder: #{$1}"
+          false
+        else
+          raise
+        end
+      end
     end
   end
 

lib/mrsk/cli/main.rb

@@ -9,6 +9,15 @@ require "mrsk/cli/server"
 require "mrsk/cli/traefik"
 
 class Mrsk::Cli::Main < Mrsk::Cli::Base
+  desc "setup", "Setup all accessories and deploy the app to servers"
+  def setup
+    print_runtime do
+      invoke "mrsk:cli:server:bootstrap"
+      invoke "mrsk:cli:accessory:boot", [ "all" ]
+      deploy
+    end
+  end
+
   desc "deploy", "Deploy the app to servers"
   def deploy
     print_runtime do
@@ -43,12 +52,13 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   def details
     invoke "mrsk:cli:traefik:details"
     invoke "mrsk:cli:app:details"
+    invoke "mrsk:cli:accessory:details", [ "all" ]
   end
 
   desc "config", "Show combined config"
   def config
     run_locally do
-      pp MRSK.config.to_h
+      puts MRSK.config.to_h.to_yaml
     end
   end
 

lib/mrsk/cli/server.rb

@@ -3,6 +3,6 @@ require "mrsk/cli/base"
 class Mrsk::Cli::Server < Mrsk::Cli::Base
   desc "bootstrap", "Ensure Docker is installed on the servers"
   def bootstrap
-    on(MRSK.hosts) { execute "which docker || (apt-get update -y && apt-get install docker.io -y)" }
+    on(MRSK.hosts + MRSK.accessory_hosts) { execute "which docker || (apt-get update -y && apt-get install docker.io -y)" }
   end
 end

lib/mrsk/cli/traefik.rb

@@ -50,7 +50,7 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
       end
     else
       since = options[:since]
-      lines = options[:lines]
+      lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
       on(MRSK.traefik_hosts) do |host|
         puts_by_host host, capture(*MRSK.traefik.logs(since: since, lines: lines, grep: grep)), type: "Traefik"

lib/mrsk/commander.rb

@@ -9,10 +9,10 @@ require "mrsk/commands/traefik"
 require "mrsk/commands/registry"
 
 class Mrsk::Commander
-  attr_accessor :config_file, :destination, :verbose, :version
+  attr_accessor :config_file, :destination, :verbosity, :version
 
-  def initialize(config_file: nil, destination: nil, verbose: false)
-    @config_file, @destination, @verbose = config_file, destination, verbose
+  def initialize(config_file: nil, destination: nil, verbosity: :info)
+    @config_file, @destination, @verbosity = config_file, destination, verbosity
   end
 
   def config
@@ -48,6 +48,10 @@ class Mrsk::Commander
     specific_hosts || config.accessories.collect(&:host)
   end
 
+  def accessory_names
+    config.accessories&.collect(&:name) || []
+  end
+
 
   def app
     @app ||= Mrsk::Commands::App.new(config)
@@ -70,11 +74,11 @@ class Mrsk::Commander
   end
 
   def accessory(name)
-    (@accessories ||= {})[name] ||= Mrsk::Commands::Accessory.new(config, name: name)
+    Mrsk::Commands::Accessory.new(config, name: name)
   end
 
 
-  def verbosity(level) 
+  def with_verbosity(level) 
     old_level = SSHKit.config.output_verbosity
     SSHKit.config.output_verbosity = level
     yield
@@ -91,6 +95,6 @@ class Mrsk::Commander
     def configure_sshkit_with(config)
       SSHKit::Backend::Netssh.configure { |ssh| ssh.ssh_options = config.ssh_options }
       SSHKit.config.command_map[:docker] = "docker" # No need to use /usr/bin/env, just clogs up the logs
-      SSHKit.config.output_verbosity = :debug if verbose
+      SSHKit.config.output_verbosity = verbosity
     end
 end

lib/mrsk/commands/accessory.rb

@@ -2,7 +2,7 @@ require "mrsk/commands/base"
 
 class Mrsk::Commands::Accessory < Mrsk::Commands::Base
   attr_reader :accessory_config
-  delegate :service_name, :image, :host, :port, :env_args, :volume_args, :label_args, to: :accessory_config
+  delegate :service_name, :image, :host, :port, :files, :directories, :env_args, :volume_args, :label_args, to: :accessory_config
 
   def initialize(config, name:)
     super(config)
@@ -42,8 +42,55 @@ class Mrsk::Commands::Accessory < Mrsk::Commands::Base
   def follow_logs(grep: nil)
     run_over_ssh pipe(
       docker(:logs, service_name, "-t", "-n", "10", "-f", "2>&1"),
-      ("grep '#{grep}'" if grep)
-    ).join(" "), host: host
+      (%(grep "#{grep}") if grep)
+    ).join(" ")
+  end
+
+  def exec(*command, interactive: false)
+    docker :exec,
+      ("-it" if interactive),
+      service_name,
+      *command
+  end
+
+  def run_exec(*command, interactive: false)
+    docker :run,
+      ("-it" if interactive),
+      "--rm",
+      *env_args,
+      *volume_args,
+      image,
+      *command
+  end
+
+  def run_over_ssh(command)
+    super command, host: host
+  end
+
+  def exec_over_ssh(*command)
+    run_over_ssh run_exec(*command, interactive: true).join(" ")
+  end
+
+  def bash
+    exec_over_ssh "bash"
+  end
+
+  def ensure_local_file_present(local_file)
+    if !local_file.is_a?(StringIO) && !Pathname.new(local_file).exist?
+      raise "Missing file: #{local_file}"
+    end
+  end
+
+  def make_directory_for(remote_file)
+    make_directory Pathname.new(remote_file).dirname.to_s
+  end
+
+  def make_directory(path)
+    [ :mkdir, "-p", path ]
+  end
+
+  def remove_service_directory
+    [ :rm, "-rf", service_name ]
   end
 
   def remove_container

lib/mrsk/commands/app.rb

@@ -35,16 +35,13 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   def logs(since: nil, lines: nil, grep: nil)
     pipe \
       current_container_id,
-      "xargs docker logs#{" --since #{since}" if since}#{" -n #{lines}" if lines} -t 2>&1",
+      "xargs docker logs#{" --since #{since}" if since}#{" -n #{lines}" if lines} 2>&1",
       ("grep '#{grep}'" if grep)
   end
 
   def exec(*command, interactive: false)
     docker :exec,
       ("-it" if interactive),
-      *rails_master_key_arg,
-      *config.env_args,
-      *config.volume_args,
       config.service_with_version,
       *command
   end
@@ -60,11 +57,15 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
       *command
   end
 
+  def exec_over_ssh(*command, host:)
+    run_over_ssh run_exec(*command, interactive: true).join(" "), host: host
+  end
+
   def follow_logs(host:, grep: nil)
     run_over_ssh pipe(
       current_container_id,
       "xargs docker logs -t -n 10 -f 2>&1",
-      ("grep '#{grep}'" if grep)
+      (%(grep "#{grep}") if grep)
     ).join(" "), host: host
   end
 
@@ -89,15 +90,15 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   end
 
   private
-    def exec_over_ssh(*command, host:)
-      run_over_ssh run_exec(*command, interactive: true).join(" "), host: host
-    end
-
     def service_filter
       [ "--filter", "label=service=#{config.service}" ]
     end
 
     def rails_master_key_arg
-      [ "-e", redact("RAILS_MASTER_KEY=#{config.master_key}") ]
+      if master_key = config.master_key
+        [ "-e", redact("RAILS_MASTER_KEY=#{master_key}") ]
+      else
+        []
+      end
     end
 end

lib/mrsk/commands/base.rb

@@ -8,6 +8,10 @@ module Mrsk::Commands
       @config = config
     end
 
+    def run_over_ssh(command, host:)
+      "ssh -t #{config.ssh_user}@#{host} '#{command}'"
+    end
+
     private
       def combine(*commands, by: "&&")
         commands
@@ -16,6 +20,10 @@ module Mrsk::Commands
           .tap     { |commands| commands.pop } # Remove trailing combiner
       end
 
+      def chain(*commands)
+        combine *commands, by: ";"
+      end
+
       def pipe(*commands)
         combine *commands, by: "|"
       end
@@ -23,9 +31,5 @@ module Mrsk::Commands
       def docker(*args)
         args.compact.unshift :docker
       end
-
-      def run_over_ssh(command, host:)
-        "ssh -t #{config.ssh_user}@#{host} '#{command}'"
-      end
   end
 end

lib/mrsk/commands/builder/native/remote.rb

@@ -2,13 +2,13 @@ require "mrsk/commands/builder/native"
 
 class Mrsk::Commands::Builder::Native::Remote < Mrsk::Commands::Builder::Native
   def create
-    combine \
+    chain \
       create_context,
       create_buildx
   end
 
   def remove
-    combine \
+    chain \
       remove_context,
       remove_buildx
   end
@@ -25,7 +25,7 @@ class Mrsk::Commands::Builder::Native::Remote < Mrsk::Commands::Builder::Native
   end
 
   def info
-    combine \
+    chain \
       docker(:context, :ls),
       docker(:buildx, :ls)
   end

lib/mrsk/commands/traefik.rb

@@ -9,7 +9,8 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
       "-v /var/run/docker.sock:/var/run/docker.sock",
       "traefik",
       "--providers.docker",
-      "--log.level=DEBUG"
+      "--log.level=DEBUG",
+      *cmd_args
   end
 
   def start
@@ -33,7 +34,7 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
   def follow_logs(host:, grep: nil)
     run_over_ssh pipe(
       docker(:logs, "traefik", "-t", "-n", "10", "-f", "2>&1"),
-      ("grep '#{grep}'" if grep)
+      (%(grep "#{grep}") if grep)
     ).join(" "), host: host
   end
 
@@ -44,4 +45,9 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
   def remove_image
     docker :image, :prune, "-a", "-f", "--filter", "label=org.opencontainers.image.title=Traefik"
   end
+
+  private
+    def cmd_args
+      (config.raw_config.dig(:traefik, "args") || { }).collect { |(key, value)| [ "--#{key}", value ] }.flatten
+    end
 end

lib/mrsk/configuration.rb

@@ -15,7 +15,7 @@ class Mrsk::Configuration
     def create_from(base_config_file, destination: nil, version: "missing")
       new(load_config_file(base_config_file).tap do |config|
         if destination
-          config.merge! \
+          config.deep_merge! \
             load_config_file destination_config_file(base_config_file, destination)
         end
       end, version: version)
@@ -39,7 +39,7 @@ class Mrsk::Configuration
   def initialize(raw_config, version: "missing", validate: true)
     @raw_config = ActiveSupport::InheritableOptions.new(raw_config)
     @version = version
-    ensure_required_keys_present if validate
+    valid? if validate
   end
 
 
@@ -52,7 +52,7 @@ class Mrsk::Configuration
   end
 
   def accessories
-    @accessories ||= raw_config.accessories.keys.collect { |name| Mrsk::Configuration::Assessory.new(name, config: self) }
+    @accessories ||= raw_config.accessories&.keys&.collect { |name| Mrsk::Configuration::Assessory.new(name, config: self) } || []
   end
 
   def accessory(name)
@@ -115,9 +115,17 @@ class Mrsk::Configuration
   end
 
   def master_key
-    ENV["RAILS_MASTER_KEY"] || File.read(Pathname.new(File.expand_path("config/master.key")))
+    unless raw_config.skip_master_key
+      ENV["RAILS_MASTER_KEY"] || File.read(Pathname.new(File.expand_path("config/master.key")))
+    end
   end
 
+
+  def valid?
+    ensure_required_keys_present && ensure_env_available
+  end
+
+
   def to_h
     {
       roles: role_names,
@@ -130,12 +138,14 @@ class Mrsk::Configuration
       env_args: env_args,
       volume_args: volume_args,
       ssh_options: ssh_options,
-      builder: raw_config.builder
+      builder: raw_config.builder,
+      accessories: raw_config.accessories
     }.compact
   end
 
 
   private
+    # Will raise ArgumentError if any required config keys are missing
     def ensure_required_keys_present
       %i[ service image registry servers ].each do |key|
         raise ArgumentError, "Missing required configuration for #{key}" unless raw_config[key].present?
@@ -148,6 +158,16 @@ class Mrsk::Configuration
       if raw_config.registry["password"].blank?
         raise ArgumentError, "You must specify a password for the registry in config/deploy.yml (or set the ENV variable if that's used)"
       end
+
+      true
+    end
+
+    # Will raise KeyError if any secret ENVs are missing
+    def ensure_env_available
+      env_args
+      roles.each(&:env_args)
+
+      true
     end
 
     def role_names

lib/mrsk/configuration/accessory.rb

@@ -43,8 +43,22 @@ class Mrsk::Configuration::Assessory
     argumentize_env_with_secrets env
   end
 
+  def files
+    specifics["files"]&.to_h do |local_to_remote_mapping|
+      local_file, remote_file = local_to_remote_mapping.split(":")
+      [ expand_local_file(local_file), expand_remote_file(remote_file) ]
+    end || {}
+  end
+
+  def directories
+    specifics["directories"]&.to_h do |host_to_container_mapping|
+      host_relative_path, container_path = host_to_container_mapping.split(":")
+      [ expand_host_path(host_relative_path), container_path ]
+    end || {}
+  end
+
   def volumes
-    specifics["volumes"] || []
+    specific_volumes + remote_files_as_volumes + remote_directories_as_volumes
   end
 
   def volume_args
@@ -57,4 +71,53 @@ class Mrsk::Configuration::Assessory
     def default_labels
       { "service" => service_name }
     end
+
+    def expand_local_file(local_file)
+      if local_file.end_with?("erb")
+        with_clear_env_loaded { read_dynamic_file(local_file) }
+      else
+        Pathname.new(File.expand_path(local_file)).to_s
+      end
+    end
+
+    def with_clear_env_loaded
+      (env["clear"] || env).each { |k, v| ENV[k] = v }
+      yield
+    ensure
+      (env["clear"] || env).each { |k, v| ENV.delete(k) }
+    end
+
+    def read_dynamic_file(local_file)
+      StringIO.new(ERB.new(IO.read(local_file)).result)
+    end
+
+    def expand_remote_file(remote_file)
+      service_name + remote_file
+    end
+
+    def specific_volumes
+      specifics["volumes"] || []
+    end
+
+    def remote_files_as_volumes
+      specifics["files"]&.collect do |local_to_remote_mapping|
+        _, remote_file = local_to_remote_mapping.split(":")
+        "#{service_data_directory + remote_file}:#{remote_file}"
+      end || []
+    end
+
+    def remote_directories_as_volumes
+      specifics["directories"]&.collect do |host_to_container_mapping|
+        host_relative_path, container_path = host_to_container_mapping.split(":")
+        [ expand_host_path(host_relative_path), container_path ].join(":")
+      end || []
+    end
+
+    def expand_host_path(host_relative_path)
+      "#{service_data_directory}/#{host_relative_path}"
+    end
+
+    def service_data_directory
+      "$PWD/#{service_name}"
+    end
 end

lib/mrsk/utils.rb

@@ -18,7 +18,7 @@ module Mrsk::Utils
     if (secrets = env["secret"]).present?
       argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, redacted: true) + argumentize("-e", env["clear"])
     else
-      argumentize "-e", env
+      argumentize "-e", env.fetch("clear", env)
     end
   end
 

lib/mrsk/version.rb

@@ -1,3 +1,3 @@
 module Mrsk
-  VERSION = "0.2.0"
+  VERSION = "0.4.0"
 end

mrsk.gemspec

@@ -15,4 +15,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "activesupport", ">= 7.0"
   spec.add_dependency "sshkit", "~> 1.21"
   spec.add_dependency "thor", "~> 1.2"
+  spec.add_dependency "dotenv", "~> 2.8"
 end

test/cli/accessory_test.rb

@@ -5,13 +5,30 @@ require "mrsk/cli"
 class CliAccessoryTest < ActiveSupport::TestCase
   include ActiveSupport::Testing::Stream
 
-  test "boot" do
-    ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
+  setup    { ENV["MYSQL_ROOT_PASSWORD"] = "secret123" }
+  teardown { ENV["MYSQL_ROOT_PASSWORD"] = nil }
 
+  test "upload" do
+    command = stdouted { Mrsk::Cli::Accessory.start(["upload", "mysql", "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+    
+    assert_match "test/fixtures/files/my.cnf app-mysql/etc/mysql/my.cnf", command
+  end
+
+  test "directories" do
+    command = stdouted { Mrsk::Cli::Accessory.start(["directories", "mysql", "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+    
+    assert_match "mkdir -p $PWD/app-mysql/data", command
+  end
+
+  test "remove service direcotry" do
+    command = stdouted { Mrsk::Cli::Accessory.start(["remove_service_directory", "mysql", "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+    
+    assert_match "rm -rf app-mysql", command
+  end
+
+  test "boot" do
     command = stdouted { Mrsk::Cli::Accessory.start(["boot", "mysql", "-c", "test/fixtures/deploy_with_accessories.yml"]) }
     
-    assert_match "Running docker run --name app-mysql -d --restart unless-stopped -p 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=% --volume /var/lib/mysql:/var/lib/mysql --label service=app-mysql mysql:5.7 on 1.1.1.3", command
-  ensure
-    ENV["MYSQL_ROOT_PASSWORD"] = nil
+    assert_match "Running docker run --name app-mysql -d --restart unless-stopped -p 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=% --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=app-mysql mysql:5.7 on 1.1.1.3", command
   end
 end

test/commands/app_test.rb

@@ -2,14 +2,18 @@ require "test_helper"
 require "mrsk/configuration"
 require "mrsk/commands/app"
 
-ENV["RAILS_MASTER_KEY"] = "456"
-
 class CommandsAppTest < ActiveSupport::TestCase
   setup do
+    ENV["RAILS_MASTER_KEY"] = "456"
+
     @config = { service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ] }
     @app = Mrsk::Commands::App.new Mrsk::Configuration.new(@config)
   end
 
+  teardown do
+    ENV["RAILS_MASTER_KEY"] = nil
+  end
+
   test "run" do
     assert_equal \
       [:docker, :run, "-d", "--restart unless-stopped", "--name", "app-missing", "-e", "RAILS_MASTER_KEY=456", "--label", "service=app", "--label", "role=web", "--label", "traefik.http.routers.app.rule='PathPrefix(`/`)'", "--label", "traefik.http.services.app.loadbalancer.healthcheck.path=/up", "--label", "traefik.http.services.app.loadbalancer.healthcheck.interval=1s", "--label", "traefik.http.middlewares.app.retry.attempts=3", "--label", "traefik.http.middlewares.app.retry.initialinterval=500ms", "dhh/app:missing"], @app.run
@@ -27,4 +31,11 @@ class CommandsAppTest < ActiveSupport::TestCase
       [ :docker, :run, "--rm", "-e", "RAILS_MASTER_KEY=456", "dhh/app:missing", "bin/rails", "db:setup" ],
       @app.run_exec("bin/rails", "db:setup")
   end
+
+  test "run without master key" do
+    ENV["RAILS_MASTER_KEY"] = nil
+    @app = Mrsk::Commands::App.new Mrsk::Configuration.new(@config.tap { |c| c[:skip_master_key] = true })
+
+    assert @app.run.exclude?("RAILS_MASTER_KEY=456")
+  end
 end

test/commands/traefik_test.rb

@@ -0,0 +1,23 @@
+require "test_helper"
+require "mrsk/configuration"
+require "mrsk/commands/traefik"
+
+class CommandsTraefikTest < ActiveSupport::TestCase
+  setup do
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
+    }
+  end
+
+  test "run" do
+    assert_equal \
+      [:docker, :run, "--name traefik", "-d", "--restart unless-stopped", "-p 80:80", "-v /var/run/docker.sock:/var/run/docker.sock", "traefik", "--providers.docker", "--log.level=DEBUG", "--accesslog.format", "json", "--metrics.prometheus.buckets", "0.1,0.3,1.2,5.0"],
+      new_command.run
+  end
+
+  private
+    def new_command
+      Mrsk::Commands::Traefik.new(Mrsk::Configuration.new(@config, version: "123"))
+    end
+end

test/configuration/accessory_test.rb

@@ -18,8 +18,15 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
             },
             "secret" => [
               "MYSQL_ROOT_PASSWORD"
-            ]
-          }
+            ],
+          },
+          "files" => [
+            "config/mysql/my.cnf:/etc/mysql/my.cnf",
+            "db/structure.sql:/docker-entrypoint-initdb.d/structure.sql"
+          ],
+          "directories" => [
+            "data:/var/lib/mysql"
+          ]
         },
         "redis" => {
           "image" => "redis:latest",
@@ -83,7 +90,19 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
   end
 
   test "volume args" do
-    assert_equal [], @config.accessory(:mysql).volume_args
+    assert_equal ["--volume", "$PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf", "--volume", "$PWD/app-mysql/docker-entrypoint-initdb.d/structure.sql:/docker-entrypoint-initdb.d/structure.sql", "--volume", "$PWD/app-mysql/data:/var/lib/mysql"], @config.accessory(:mysql).volume_args
     assert_equal ["--volume", "/var/lib/redis:/data"], @config.accessory(:redis).volume_args
   end
+
+  test "dynamic file expansion" do
+    @deploy[:accessories]["mysql"]["files"] << "test/fixtures/files/structure.sql.erb:/docker-entrypoint-initdb.d/structure.sql"
+    @config = Mrsk::Configuration.new(@deploy)
+
+    assert_match "This was dynamically expanded", @config.accessory(:mysql).files.keys[2].read
+    assert_match "%", @config.accessory(:mysql).files.keys[2].read
+  end
+
+  test "directories" do
+    assert_equal({"$PWD/app-mysql/data"=>"/var/lib/mysql"}, @config.accessory(:mysql).directories)
+  end
 end

test/configuration_test.rb

@@ -1,10 +1,10 @@
 require "test_helper"
 require "mrsk/configuration"
 
-ENV["RAILS_MASTER_KEY"] = "456"
-
 class ConfigurationTest < ActiveSupport::TestCase
   setup do
+    ENV["RAILS_MASTER_KEY"] = "456"
+
     @deploy = {
       service: "app", image: "dhh/app",
       registry: { "username" => "dhh", "password" => "secret" },
@@ -21,6 +21,10 @@ class ConfigurationTest < ActiveSupport::TestCase
     @config_with_roles = Mrsk::Configuration.new(@deploy_with_roles)
   end
 
+  teardown do
+    ENV["RAILS_MASTER_KEY"] = nil
+  end
+
   test "ensure valid keys" do
     assert_raise(ArgumentError) do
       Mrsk::Configuration.new(@deploy.tap { _1.delete(:service) })
@@ -101,6 +105,14 @@ class ConfigurationTest < ActiveSupport::TestCase
     ENV["PASSWORD"] = nil
   end
 
+  test "env args with only clear" do
+    config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
+      env: { "clear" => { "PORT" => "3000" } }
+    }) })
+
+    assert_equal [ "-e", "PORT=3000" ], config.env_args
+  end
+
   test "env args with only secrets" do
     ENV["PASSWORD"] = "secret123"
     config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
@@ -114,15 +126,17 @@ class ConfigurationTest < ActiveSupport::TestCase
   end
 
   test "env args with missing secret" do
-    config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
-      env: { "secret" => [ "PASSWORD" ] }
-    }) })
-
     assert_raises(KeyError) do
-      assert_equal [ "-e", "PASSWORD=secret123" ], config.env_args
+      config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
+        env: { "secret" => [ "PASSWORD" ] }
+      }) })
     end
   end
 
+  test "valid config" do
+    assert @config.valid?
+  end
+
   test "ssh options" do
     assert_equal "root", @config.ssh_options[:user]
 
@@ -134,6 +148,11 @@ class ConfigurationTest < ActiveSupport::TestCase
     assert_equal "456", @config.master_key
   end
 
+  test "skip master key" do
+    config = Mrsk::Configuration.new(@deploy.tap { |c| c[:skip_master_key] = true })
+    assert_nil @config.master_key
+  end
+
   test "volume_args" do
     assert_equal ["--volume", "/local/path:/container/path"], @config.volume_args
   end

test/fixtures/deploy_with_accessories.yml

@@ -17,11 +17,13 @@ accessories:
         MYSQL_ROOT_HOST: '%'
       secret:
         - MYSQL_ROOT_PASSWORD
-    volumes:
-      - /var/lib/mysql:/var/lib/mysql
+    files:
+      - test/fixtures/files/my.cnf:/etc/mysql/my.cnf
+    directories:
+      - data:/var/lib/mysql
   redis:
     image: redis:latest
     host: 1.1.1.4
     port: 6379
-    volumes:
-      - /var/lib/redis:/data
+    directories:
+      - data:/data

test/fixtures/files/my.cnf

@@ -0,0 +1 @@
+# MySQL Config

test/fixtures/files/structure.sql.erb

@@ -0,0 +1,2 @@
+<%= "This was dynamically expanded" %>
+<%= ENV["MYSQL_ROOT_HOST"] %>