Bump version for 0.6.0

You can use MRSK with something other than Rails.

.gitignore

@@ -1,2 +1,4 @@
 .byebug_history
 *.gem
+coverage/*
+.DS_Store

Gemfile

@@ -4,4 +4,5 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 gemspec
 
 gem "debug"
+gem "mocha"
 gem "railties"

Gemfile.lock

@@ -1,10 +1,12 @@
 PATH
   remote: .
   specs:
-    mrsk (0.2.0)
+    mrsk (0.6.0)
       activesupport (>= 7.0)
+      dotenv (~> 2.8)
       sshkit (~> 1.21)
       thor (~> 1.2)
+      zeitwerk (~> 2.5)
 
 GEM
   remote: https://rubygems.org/
@@ -33,6 +35,7 @@ GEM
     debug (1.7.1)
       irb (>= 1.5.0)
       reline (>= 0.3.1)
+    dotenv (2.8.1)
     erubi (1.12.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
@@ -44,6 +47,8 @@ GEM
       nokogiri (>= 1.5.9)
     method_source (1.0.0)
     minitest (5.17.0)
+    mocha (2.0.2)
+      ruby2_keywords (>= 0.0.5)
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
     net-ssh (7.0.1)
@@ -72,6 +77,7 @@ GEM
     rake (13.0.6)
     reline (0.3.2)
       io-console (~> 0.5)
+    ruby2_keywords (0.0.5)
     sshkit (1.21.3)
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
@@ -86,10 +92,12 @@ PLATFORMS
   arm64-darwin-22
   x86_64-darwin-20
   x86_64-darwin-21
+  x86_64-darwin-22
   x86_64-linux
 
 DEPENDENCIES
   debug
+  mocha
   mrsk!
   railties
 

README.md

@@ -1,10 +1,10 @@
 # MRSK
 
-MRSK deploys Rails apps in containers to servers running Docker with zero downtime. It uses the dynamic reverse-proxy Traefik to hold requests while the new application container is started and the old one is stopped. It works seamlessly across multiple hosts, using SSHKit to execute commands.
+MRSK deploys web apps in containers to servers running Docker with zero downtime. It uses the dynamic reverse-proxy Traefik to hold requests while the new application container is started and the old one is stopped. It works seamlessly across multiple hosts, using SSHKit to execute commands.
 
 ## Installation
 
-Install MRSK globally with `gem install mrsk`. Then, inside your app directory, run `mrsk install`. Now edit the new file `config/deploy.yml`. It could look as simple as this:
+Install MRSK globally with `gem install mrsk`. Then, inside your app directory, run `mrsk init` (or `mrsk init --bundle` within Rails apps where you want a bin/mrsk binstub). Now edit the new file `config/deploy.yml`. It could look as simple as this:
 
 ```yaml
 service: hey
@@ -15,12 +15,17 @@ servers:
 registry:
   username: registry-user-name
   password: <%= ENV.fetch("MRSK_REGISTRY_PASSWORD") %>
+env:
+  secret:
+    - RAILS_MASTER_KEY
 ```
 
-Now you're ready to deploy a multi-arch image to the servers:
+Then edit your `.env` file to add your registry password as `MRSK_REGISTRY_PASSWORD` (and your `RAILS_MASTER_KEY` for production with a Rails app).
+
+Now you're ready to deploy to the servers:
 
 ```
-MRSK_REGISTRY_PASSWORD=pw mrsk deploy
+mrsk deploy
 ```
 
 This will:
@@ -46,6 +51,15 @@ Kubernetes is a beast. Running it yourself on your own hardware is not for the f
 
 ## Configuration
 
+### Using .env file to load required environment variables
+
+MRSK uses [dotenv](https://github.com/bkeepers/dotenv) to automatically load environment variables set in the `.env` file present in the application root. This file can be used to set variables like `MRSK_REGISTRY_PASSWORD` or database passwords. But for this reason you must ensure that .env files are not checked into Git or included in your Dockerfile! The format is just key-value like:
+
+```bash
+MRSK_REGISTRY_PASSWORD=pw
+DB_PASSWORD=secret123
+```
+
 ### Using another registry than Docker Hub
 
 The default registry is Docker Hub, but you can change it using `registry/server`:
@@ -59,10 +73,27 @@ registry:
 
 ### Using a different SSH user than root
 
-The default SSH user is root, but you can change it using `ssh_user`:
+The default SSH user is root, but you can change it using `ssh/user`:
 
 ```yaml
-ssh_user: app
+ssh:
+  user: app
+```
+
+### Using a proxy SSH host
+
+If you need to connect to server through a proxy host, you can use `ssh/proxy`:
+
+```yaml
+ssh:
+  proxy: "192.168.0.1" # defaults to root as the user
+```
+
+Or with specific user:
+
+```yaml
+ssh:
+  proxy: "app@192.168.0.1"
 ```
 
 ### Using env variables
@@ -226,6 +257,18 @@ RUN --mount=type=secret,id=GITHUB_TOKEN \
   bundle install
 ```
 
+### Using command arguments for Traefik
+
+You can customize the traefik command line:
+
+```yaml
+traefik:
+  accesslog: true
+  accesslog.format: json
+  metrics.prometheus: true
+  metrics.prometheus.buckets: 0.1,0.3,1.2,5.0
+```
+
 ### Configuring build args for new images
 
 Build arguments that aren't secret can also be configured:
@@ -271,11 +314,28 @@ accessories:
 
 Now run `mrsk accessory start mysql` to start the MySQL server on 1.1.1.3. See `mrsk accessory` for all the commands possible.
 
+### Using a generated .env file
+
+If you're using a centralized secret store, like 1Password, you can create `.env.erb` as a template which looks up the secrets. Example of a .env.erb file:
+
+```erb
+<% if (session_token = `op signin --account my-one-password-account --raw`.strip) != "" %># Generated by mrsk envify
+GITHUB_TOKEN=<%= `gh config get -h github.com oauth_token`.strip %>
+MRSK_REGISTRY_PASSWORD=<%= `op read "op://Vault/Docker Hub/password" -n --session  #{session_token}` %>
+RAILS_MASTER_KEY=<%= `op read "op://Vault/My App/RAILS_MASTER_SECRET" -n --session #{session_token}` %>
+MYSQL_ROOT_PASSWORD=<%= `op read "op://Vault/My App/MYSQL_ROOT_PASSWORD" -n --session #{session_token}` %>
+<% else raise ArgumentError, "Session token missing" end %>
+```
+
+This template can safely be checked into git. Then everyone deploying the app can run `mrsk envify` when they setup the app for the first time or passwords change to get the correct `.env` file.
+
+If you need separate env variables for different destinations, you can set them with `.env.destination.erb` for the template, which will generate `.env.staging` when run with `mrsk envify -d staging`.
+
 ## Commands
 
-### Running remote execution and runners
+### Running commands on servers
 
-If you need to execute commands inside the Rails containers, you can use `mrsk app exec` and `mrsk app runner`. Examples:
+You can execute one-off commands on the servers:
 
 ```bash
 # Runs command on all servers
@@ -318,13 +378,25 @@ Database adapter          sqlite3
 Database schema version   20221231233303
 
 # Run Rails runner on primary server
-mrsk app runner -p 'puts Rails.application.config.time_zone'
+mrsk app exec -p 'bin/rails runner "puts Rails.application.config.time_zone"'
 UTC
 ```
 
-### Running a Rails console
+### Running interactive commands over SSH
+
+You can run interactive commands, like a Rails console or a bash session, on a server (default is primary, use `--hosts` to connect to another):
+
+```bash
+# Starts a bash session in a new container made from the most recent app image
+mrsk app exec -i bash
+
+# Starts a bash session in the currently running container for the app
+mrsk app exec -i --reuse bash
+
+# Starts a Rails console in a new container made from the most recent app image
+mrsk app exec -i 'bin/rails console'
+```
 
-If you need to interact with the production console for the app, you can use `mrsk app console`, which will start a Rails console session on the primary host. You can start the console on a different host using `mrsk app console --host 192.168.0.2`. Be mindful that this is a live wire! Any changes made to the production database will take effect immeditately.
 
 ### Running details to see state of containers
 

bin/mrsk

@@ -1,5 +1,16 @@
 #!/usr/bin/env ruby
 
-require "mrsk/cli"
+# Prevent failures from being reported twice.
+Thread.report_on_exception = false
 
-Mrsk::Cli::Main.start(ARGV)
+require "mrsk"
+
+begin
+  Mrsk::Cli::Main.start(ARGV)
+rescue SSHKit::Runner::ExecuteError => e
+  puts "  \e[31mERROR (#{e.cause.class}): #{e.cause.message}\e[0m"
+  puts e.cause.backtrace if ENV["VERBOSE"]
+rescue => e
+  puts "  \e[31mERROR (#{e.class}): #{e.message}\e[0m"
+  puts e.backtrace if ENV["VERBOSE"]
+end

lib/mrsk.rb

@@ -1,5 +1,9 @@
 module Mrsk
 end
 
-require "mrsk/version"
-require "mrsk/commander"
+require "zeitwerk"
+
+loader = Zeitwerk::Loader.for_gem
+loader.ignore("#{__dir__}/mrsk/sshkit_with_ext.rb")
+loader.setup
+loader.eager_load # We need all commands loaded.

lib/mrsk/cli.rb

@@ -1,9 +1,5 @@
-require "mrsk"
-
 module Mrsk::Cli
 end
 
 # SSHKit uses instance eval, so we need a global const for ergonomics
 MRSK = Mrsk::Commander.new
-
-require "mrsk/cli/main"

lib/mrsk/cli/accessory.rb

@@ -1,41 +1,128 @@
-require "mrsk/cli/base"
-
 class Mrsk::Cli::Accessory < Mrsk::Cli::Base
-  desc "boot [NAME]", "Boot accessory service on host"
+  desc "boot [NAME]", "Boot accessory service on host (use NAME=all to boot all accessories)"
   def boot(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.run }
+    if name == "all"
+      MRSK.accessory_names.each { |accessory_name| boot(accessory_name) }
+    else
+      with_accessory(name) do |accessory|
+        directories(name)
+        upload(name)
+
+        on(accessory.host) do
+          execute *MRSK.auditor.record("accessory #{name} boot"), verbosity: :debug
+          execute *accessory.run
+        end
+      end
+    end
+  end
+
+  desc "upload [NAME]", "Upload accessory files to host"
+  def upload(name)
+    with_accessory(name) do |accessory|
+      on(accessory.host) do
+        execute *MRSK.auditor.record("accessory #{name} upload files"), verbosity: :debug
+
+        accessory.files.each do |(local, remote)|
+          accessory.ensure_local_file_present(local)
+
+          execute *accessory.make_directory_for(remote)
+          upload! local, remote
+          execute :chmod, "755", remote
+        end
+      end
+    end
+  end
+
+  desc "directories [NAME]", "Create accessory directories on host"
+  def directories(name)
+    with_accessory(name) do |accessory|
+      on(accessory.host) do
+        execute *MRSK.auditor.record("accessory #{name} create directories"), verbosity: :debug
+
+        accessory.directories.keys.each do |host_path|
+          execute *accessory.make_directory(host_path)
+        end
+      end
+    end
   end
 
   desc "reboot [NAME]", "Reboot accessory on host (stop container, remove container, start new container)"
   def reboot(name)
-    invoke :stop, [ name ]
-    invoke :remove_container, [ name ]
-    invoke :boot, [ name ]
+    with_accessory(name) do |accessory|
+      stop(name)
+      remove_container(name)
+      boot(name)
+    end
   end
 
   desc "start [NAME]", "Start existing accessory on host"
   def start(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.start }
+    with_accessory(name) do |accessory|
+      on(accessory.host) do
+        execute *MRSK.auditor.record("accessory #{name} start"), verbosity: :debug
+        execute *accessory.start
+      end
+    end
   end
 
   desc "stop [NAME]", "Stop accessory on host"
   def stop(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.stop }
+    with_accessory(name) do |accessory|
+      on(accessory.host) do
+        execute *MRSK.auditor.record("accessory #{name} stop"), verbosity: :debug
+        execute *accessory.stop, raise_on_non_zero_exit: false
+      end
+    end
   end
 
   desc "restart [NAME]", "Restart accessory on host"
   def restart(name)
-    invoke :stop, [ name ]
-    invoke :start, [ name ]
+    with_accessory(name) do
+      stop(name)
+      start(name)
+    end
   end
 
-  desc "details [NAME]", "Display details about accessory on host"
+  desc "details [NAME]", "Display details about accessory on host (use NAME=all to boot all accessories)"
   def details(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { puts capture_with_info(*accessory.info) }
+    if name == "all"
+      MRSK.accessory_names.each { |accessory_name| details(accessory_name) }
+    else
+      with_accessory(name) do |accessory|
+        on(accessory.host) { puts capture_with_info(*accessory.info) }
+      end
+    end
+  end
+
+  desc "exec [NAME] [CMD]", "Execute a custom command on servers"
+  option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
+  option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
+  def exec(name, cmd)
+    with_accessory(name) do |accessory|
+      case
+      when options[:interactive] && options[:reuse]
+        say "Launching interactive command with via SSH from existing container...", :magenta
+        run_locally { exec accessory.execute_in_existing_container_over_ssh(cmd) }
+
+      when options[:interactive]
+        say "Launching interactive command via SSH from new container...", :magenta
+        run_locally { exec accessory.execute_in_new_container_over_ssh(cmd) }
+
+      when options[:reuse]
+        say "Launching command from existing container...", :magenta
+        on(accessory.host) do
+          execute *MRSK.auditor.record("accessory #{name} cmd '#{cmd}'"), verbosity: :debug
+          capture_with_info(*accessory.execute_in_existing_container(cmd))
+        end
+
+      else
+        say "Launching command from new container...", :magenta
+        on(accessory.host) do
+          execute *MRSK.auditor.record("accessory #{name} cmd '#{cmd}'"), verbosity: :debug
+          capture_with_info(*accessory.execute_in_new_container(cmd))
+        end
+      end
+    end
   end
 
   desc "logs [NAME]", "Show log lines from accessory on host"
@@ -44,42 +131,84 @@ class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
   option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
   def logs(name)
-    accessory = MRSK.accessory(name)
+    with_accessory(name) do |accessory|
+      grep = options[:grep]
 
-    grep = options[:grep]
+      if options[:follow]
+        run_locally do
+          info "Following logs on #{accessory.host}..."
+          info accessory.follow_logs(grep: grep)
+          exec accessory.follow_logs(grep: grep)
+        end
+      else
+        since = options[:since]
+        lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
-    if options[:follow]
-      run_locally do
-        info "Following logs on #{accessory.host}..."
-        info accessory.follow_logs(grep: grep)
-        exec accessory.follow_logs(grep: grep)
-      end
-    else
-      since = options[:since]
-      lines = options[:lines]
-
-      on(accessory.host) do
-        puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep))
+        on(accessory.host) do
+          puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep))
+        end
       end
     end
   end
 
-  desc "remove [NAME]", "Remove accessory container and image from host"
+  desc "remove [NAME]", "Remove accessory container and image from host (use NAME=all to boot all accessories)"
   def remove(name)
-    invoke :stop, [ name ]
-    invoke :remove_container, [ name ]
-    invoke :remove_image, [ name ]
+    if name == "all"
+      MRSK.accessory_names.each { |accessory_name| remove(accessory_name) }
+    else
+      with_accessory(name) do
+        stop(name)
+        remove_container(name)
+        remove_image(name)
+        remove_service_directory(name)
+      end
+    end
   end
 
   desc "remove_container [NAME]", "Remove accessory container from host"
   def remove_container(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.remove_container }
+    with_accessory(name) do |accessory|
+      on(accessory.host) do
+        execute *MRSK.auditor.record("accessory #{name} remove container"), verbosity: :debug
+        execute *accessory.remove_container
+      end
+    end
   end
 
-  desc "remove_container [NAME]", "Remove accessory image from servers"
+  desc "remove_image [NAME]", "Remove accessory image from host"
   def remove_image(name)
-    accessory = MRSK.accessory(name)
-    on(accessory.host) { execute *accessory.remove_image }
+    with_accessory(name) do |accessory|
+      on(accessory.host) do
+        execute *MRSK.auditor.record("accessory #{name} remove image"), verbosity: :debug
+        execute *accessory.remove_image
+      end
+    end
   end
+
+  desc "remove_service_directory [NAME]", "Remove accessory directory used for uploaded files and data directories from host"
+  def remove_service_directory(name)
+    with_accessory(name) do |accessory|
+      on(accessory.host) do
+        execute *MRSK.auditor.record("accessory #{name} remove service directory"), verbosity: :debug
+        execute *accessory.remove_service_directory
+      end
+    end
+  end
+
+  private
+    def with_accessory(name)
+      if accessory = MRSK.accessory(name)
+        yield accessory
+      else
+        error_on_missing_accessory(name)
+      end
+    end
+
+    def error_on_missing_accessory(name)
+      options = MRSK.accessory_names.presence
+
+      error \
+        "No accessory by the name of '#{name}'" +
+        (options ? " (options: #{options.to_sentence})" : "")
+    end
 end

lib/mrsk/cli/app.rb

@@ -1,82 +1,109 @@
-require "mrsk/cli/base"
-
 class Mrsk::Cli::App < Mrsk::Cli::Base
-  desc "boot", "Boot app on servers (or start them if they've already been booted)"
+  desc "boot", "Boot app on servers (or reboot app if already running)"
   def boot
-    MRSK.config.roles.each do |role|
-      on(role.hosts) do |host|
-        begin
-          execute *MRSK.app.run(role: role.name)
-        rescue SSHKit::Command::Failed => e
-          if e.message =~ /already in use/
-            error "Container with same version already deployed on #{host}, starting that instead"
-            execute *MRSK.app.start, host: host
-          else
-            raise
+    cli = self
+
+    say "Ensure no other version of the app is running...", :magenta
+    stop
+
+    say "Get most recent version available as an image...", :magenta unless options[:version]
+    using_version(options[:version] || most_recent_version_available) do |version|
+      say "Start container with version #{version} (or reboot if already running)...", :magenta
+
+      MRSK.config.roles.each do |role|
+        on(role.hosts) do |host|
+          execute *MRSK.auditor.record("app boot version #{version}"), verbosity: :debug
+
+          begin
+            execute *MRSK.app.run(role: role.name)
+          rescue SSHKit::Command::Failed => e
+            if e.message =~ /already in use/
+              error "Rebooting container with same version already deployed on #{host}"
+
+              cli.remove_container version
+              execute *MRSK.app.run(role: role.name)
+            else
+              raise
+            end
           end
         end
       end
     end
   end
-  
+
   desc "start", "Start existing app on servers (use --version=<git-hash> to designate specific version)"
-  option :version, desc: "Defaults to the most recent git-hash in local repository"
   def start
-    if (version = options[:version]).present?
-      on(MRSK.hosts) { execute *MRSK.app.start(version: version) }
-    else
-      on(MRSK.hosts) { execute *MRSK.app.start, raise_on_non_zero_exit: false }
+    on(MRSK.hosts) do
+      execute *MRSK.auditor.record("app start version #{MRSK.version}"), verbosity: :debug
+      execute *MRSK.app.start, raise_on_non_zero_exit: false
     end
   end
-  
+
   desc "stop", "Stop app on servers"
   def stop
-    on(MRSK.hosts) { execute *MRSK.app.stop, raise_on_non_zero_exit: false }
+    on(MRSK.hosts) do
+      execute *MRSK.auditor.record("app stop"), verbosity: :debug
+      execute *MRSK.app.stop, raise_on_non_zero_exit: false
+    end
   end
-  
+
   desc "details", "Display details about app containers"
   def details
     on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.info) }
   end
-  
+
   desc "exec [CMD]", "Execute a custom command on servers"
-  option :run, type: :boolean, default: false, desc: "Start a new container to run the command rather than reusing existing"
+  option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
+  option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
   def exec(cmd)
-    runner = options[:run] ? :run_exec : :exec
-    on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.send(runner, cmd)) }
-  end
+    case
+    when options[:interactive] && options[:reuse]
+      say "Get current version of running container...", :magenta unless options[:version]
+      using_version(options[:version] || current_running_version) do |version|
+        say "Launching interactive command with version #{version} via SSH from existing container on #{MRSK.primary_host}...", :magenta
+        run_locally { exec MRSK.app.execute_in_existing_container_over_ssh(cmd, host: MRSK.primary_host) }
+      end
 
-  desc "console", "Start Rails Console on primary host (or specific host set by --hosts)"
-  def console
-    run_locally do
-      info "Launching Rails console on #{MRSK.primary_host}"
-      exec MRSK.app.console(host: MRSK.primary_host)
+    when options[:interactive]
+      say "Get most recent version available as an image...", :magenta unless options[:version]
+      using_version(options[:version] || most_recent_version_available) do |version|
+        say "Launching interactive command with version #{version} via SSH from new container on #{MRSK.primary_host}...", :magenta
+        run_locally { exec MRSK.app.execute_in_new_container_over_ssh(cmd, host: MRSK.primary_host) }
+      end
+
+    when options[:reuse]
+      say "Get current version of running container...", :magenta unless options[:version]
+      using_version(options[:version] || current_running_version) do |version|
+        say "Launching command with version #{version} from existing container...", :magenta
+
+        on(MRSK.hosts) do |host|
+          execute *MRSK.auditor.record("app cmd '#{cmd}' with version #{version}"), verbosity: :debug
+          puts_by_host host, capture_with_info(*MRSK.app.execute_in_existing_container(cmd))
+        end
+      end
+
+    else
+      say "Get most recent version available as an image...", :magenta unless options[:version]
+      using_version(options[:version] || most_recent_version_available) do |version|
+        say "Launching command with version #{version} from new container...", :magenta
+        on(MRSK.hosts) do |host|
+          execute *MRSK.auditor.record("app cmd '#{cmd}' with version #{version}"), verbosity: :debug
+          puts_by_host host, capture_with_info(*MRSK.app.execute_in_new_container(cmd))
+        end
+      end
     end
   end
 
-  desc "bash", "Start a bash session on primary host (or specific host set by --hosts)"
-  def bash
-    run_locally do
-      info "Launching bash session on #{MRSK.primary_host}"
-      exec MRSK.app.bash(host: MRSK.primary_host)
-    end
-  end
-
-  desc "runner [EXPRESSION]", "Execute Rails runner with given expression"
-  def runner(expression)
-    on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.exec("bin/rails", "runner", "'#{expression}'")) }
-  end
-
   desc "containers", "List all the app containers currently on servers"
   def containers
     on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_containers) }
   end
 
-  desc "current", "Return the current running container ID"
-  def current
-    on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.current_container_id) }
+  desc "images", "List all the app images currently on servers"
+  def images
+    on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.list_images) }
   end
-  
+
   desc "logs", "Show lines from app on servers"
   option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
   option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server"
@@ -95,7 +122,7 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
       end
     else
       since = options[:since]
-      lines = options[:lines]
+      lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
       on(MRSK.hosts) do |host|
         begin
@@ -108,16 +135,64 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
   end
 
   desc "remove", "Remove app containers and images from servers"
-  option :only, default: "", desc: "Use 'containers' or 'images'"
   def remove
-    case options[:only]
-    when "containers"
-      on(MRSK.hosts) { execute *MRSK.app.remove_containers }
-    when "images"
-      on(MRSK.hosts) { execute *MRSK.app.remove_images }
-    else
-      on(MRSK.hosts) { execute *MRSK.app.remove_containers }
-      on(MRSK.hosts) { execute *MRSK.app.remove_images }
+    remove_containers
+    remove_images
+  end
+
+  desc "remove_container [VERSION]", "Remove app container with given version from servers"
+  def remove_container(version)
+    on(MRSK.hosts) do
+      execute *MRSK.auditor.record("app remove container #{version}"), verbosity: :debug
+      execute *MRSK.app.remove_container(version: version)
     end
   end
+
+  desc "remove_containers", "Remove all app containers from servers"
+  def remove_containers
+    on(MRSK.hosts) do
+      execute *MRSK.auditor.record("app remove containers"), verbosity: :debug
+      execute *MRSK.app.remove_containers
+    end
+  end
+
+  desc "remove_images", "Remove all app images from servers"
+  def remove_images
+    on(MRSK.hosts) do
+      execute *MRSK.auditor.record("app remove images"), verbosity: :debug
+      execute *MRSK.app.remove_images
+    end
+  end
+
+  desc "current_version", "Shows the version currently running"
+  def current_version
+    on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.current_running_version).strip }
+  end
+
+  private
+    def using_version(new_version)
+      if new_version
+        begin
+          old_version = MRSK.config.version
+          MRSK.config.version = new_version
+          yield new_version
+        ensure
+          MRSK.config.version = old_version
+        end
+      else
+        yield MRSK.config.version
+      end
+    end
+
+    def most_recent_version_available(host: MRSK.primary_host)
+      version = nil
+      on(host) { version = capture_with_info(*MRSK.app.most_recent_version_from_available_images).strip }
+      version.presence
+    end
+
+    def current_running_version(host: MRSK.primary_host)
+      version = nil
+      on(host) { version = capture_with_info(*MRSK.app.current_running_version).strip }
+      version.presence
+    end
 end

lib/mrsk/cli/base.rb

@@ -1,4 +1,5 @@
 require "thor"
+require "dotenv"
 require "mrsk/sshkit_with_ext"
 
 module Mrsk::Cli
@@ -8,6 +9,7 @@ module Mrsk::Cli
     def self.exit_on_failure?() true end
 
     class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging"
+    class_option :quiet, type: :boolean, aliases: "-q", desc: "Minimal logging"
 
     class_option :version, desc: "Run commands against a specific app version"
 
@@ -20,20 +22,37 @@ module Mrsk::Cli
 
     def initialize(*)
       super
+      load_envs
       initialize_commander(options)
     end
 
     private
+      def load_envs
+        if destination = options[:destination]
+          Dotenv.load(".env.#{destination}", ".env")
+        else
+          Dotenv.load(".env")
+        end
+      end
+
       def initialize_commander(options)
         MRSK.tap do |commander|
           commander.config_file = Pathname.new(File.expand_path(options[:config_file]))
           commander.destination = options[:destination]
-          commander.verbose     = options[:verbose]
           commander.version     = options[:version]
 
           commander.specific_hosts    = options[:hosts]&.split(",")
           commander.specific_roles    = options[:roles]&.split(",")
           commander.specific_primary! if options[:primary]
+
+          if options[:verbose]
+            ENV["VERBOSE"] = "1" # For backtraces via cli/start
+            commander.verbosity = :debug
+          end
+
+          if options[:quiet]
+            commander.verbosity = :error
+          end
         end
       end
 

lib/mrsk/cli/build.rb

@@ -1,5 +1,3 @@
-require "mrsk/cli/base"
-
 class Mrsk::Cli::Build < Mrsk::Cli::Base
   desc "deliver", "Deliver a newly built app image to servers"
   def deliver
@@ -9,29 +7,47 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
 
   desc "push", "Build locally and push app image to registry"
   def push
-    verbose = options[:verbose]
+    cli = self
 
-    run_locally do 
+    run_locally do
       begin
-        MRSK.verbosity(:debug) { execute *MRSK.builder.push }
+        MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
       rescue SSHKit::Command::Failed => e
-        error "Missing compatible builder, so creating a new one first"
-        execute *MRSK.builder.create
-        MRSK.verbosity(:debug) { execute *MRSK.builder.push }
+        if e.message =~ /(no builder)|(no such file or directory)/
+          error "Missing compatible builder, so creating a new one first"
+
+          if cli.create
+            MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
+          end
+        else
+          raise
+        end
       end
     end
   end
 
   desc "pull", "Pull app image from the registry onto servers"
   def pull
-    on(MRSK.hosts) { execute *MRSK.builder.pull }
+    on(MRSK.hosts) do
+      execute *MRSK.auditor.record("build pull image #{MRSK.version}"), verbosity: :debug
+      execute *MRSK.builder.pull
+    end
   end
 
   desc "create", "Create a local build setup"
   def create
     run_locally do
-      debug "Using builder: #{MRSK.builder.name}"
-      execute *MRSK.builder.create
+      begin
+        debug "Using builder: #{MRSK.builder.name}"
+        execute *MRSK.builder.create
+      rescue SSHKit::Command::Failed => e
+        if e.message =~ /stderr=(.*)/
+          error "Couldn't create remote builder: #{$1}"
+          false
+        else
+          raise
+        end
+      end
     end
   end
 

lib/mrsk/cli/main.rb

@@ -1,23 +1,31 @@
-require "mrsk/cli/base"
-
-require "mrsk/cli/accessory"
-require "mrsk/cli/app"
-require "mrsk/cli/build"
-require "mrsk/cli/prune"
-require "mrsk/cli/registry"
-require "mrsk/cli/server"
-require "mrsk/cli/traefik"
-
 class Mrsk::Cli::Main < Mrsk::Cli::Base
+  desc "setup", "Setup all accessories and deploy the app to servers"
+  def setup
+    print_runtime do
+      invoke "mrsk:cli:server:bootstrap"
+      invoke "mrsk:cli:accessory:boot", [ "all" ]
+      deploy
+    end
+  end
+
   desc "deploy", "Deploy the app to servers"
   def deploy
     print_runtime do
+      say "Ensure Docker is installed...", :magenta
       invoke "mrsk:cli:server:bootstrap"
+
+      say "Log into image registry...", :magenta
       invoke "mrsk:cli:registry:login"
+
+      say "Build and push app image...", :magenta
       invoke "mrsk:cli:build:deliver"
+
+      say "Ensure Traefik is running...", :magenta
       invoke "mrsk:cli:traefik:boot"
-      invoke "mrsk:cli:app:stop"
+
       invoke "mrsk:cli:app:boot"
+
+      say "Prune old containers and images...", :magenta
       invoke "mrsk:cli:prune:all"
     end
   end
@@ -25,17 +33,23 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "redeploy", "Deploy new version of the app to servers (without bootstrapping servers, starting Traefik, pruning, and registry login)"
   def redeploy
     print_runtime do
+      say "Build and push app image...", :magenta
       invoke "mrsk:cli:build:deliver"
-      invoke "mrsk:cli:app:stop"
+
       invoke "mrsk:cli:app:boot"
     end
   end
 
-  desc "rollback [VERSION]", "Rollback the app to VERSION (that must already be on servers)"
+  desc "rollback [VERSION]", "Rollback the app to VERSION"
   def rollback(version)
+    MRSK.version = version
+
+    cli = self
+
+    cli.say "Stop current version, then start version #{version}...", :magenta
     on(MRSK.hosts) do
       execute *MRSK.app.stop, raise_on_non_zero_exit: false
-      execute *MRSK.app.start(version: version)
+      execute *MRSK.app.start
     end
   end
 
@@ -43,31 +57,46 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   def details
     invoke "mrsk:cli:traefik:details"
     invoke "mrsk:cli:app:details"
+    invoke "mrsk:cli:accessory:details", [ "all" ]
+  end
+
+  desc "audit", "Show audit log from servers"
+  def audit
+    on(MRSK.hosts) do |host|
+      puts_by_host host, capture_with_info(*MRSK.auditor.reveal)
+    end
   end
 
   desc "config", "Show combined config"
   def config
     run_locally do
-      pp MRSK.config.to_h
+      puts MRSK.config.to_h.to_yaml
     end
   end
 
-  desc "install", "Create config stub in config/deploy.yml and binstub in bin/mrsk"
-  option :skip_binstub, type: :boolean, default: false, desc: "Skip adding MRSK to the Gemfile and creating bin/mrsk binstub"
-  def install
+  desc "init", "Create config stub in config/deploy.yml and env stub in .env"
+  option :bundle, type: :boolean, default: false, desc: "Add MRSK to the Gemfile and create a bin/mrsk binstub"
+  def init
     require "fileutils"
 
     if (deploy_file = Pathname.new(File.expand_path("config/deploy.yml"))).exist?
       puts "Config file already exists in config/deploy.yml (remove first to create a new one)"
     else
+      FileUtils.mkdir_p deploy_file.dirname
       FileUtils.cp_r Pathname.new(File.expand_path("templates/deploy.yml", __dir__)), deploy_file
       puts "Created configuration file in config/deploy.yml"
     end
 
-    unless options[:skip_binstub]
+    unless (deploy_file = Pathname.new(File.expand_path(".env"))).exist?
+      FileUtils.cp_r Pathname.new(File.expand_path("templates/template.env", __dir__)), deploy_file
+      puts "Created .env file"
+    end
+
+    if options[:bundle]
       if (binstub = Pathname.new(File.expand_path("bin/mrsk"))).exist?
         puts "Binstub already exists in bin/mrsk (remove first to create a new one)"
       else
+        puts "Adding MRSK to Gemfile and bundle..."
         `bundle add mrsk`
         `bundle binstubs mrsk`
         puts "Created binstub file in bin/mrsk"
@@ -75,6 +104,15 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
     end
   end
 
+  desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)"
+  def envify
+    if destination = options[:destination]
+      File.write(".env.#{destination}", ERB.new(IO.read(Pathname.new(File.expand_path(".env.#{destination}.erb")))).result)
+    else
+      File.write(".env", ERB.new(IO.read(Pathname.new(File.expand_path(".env.erb")))).result)
+    end
+  end
+
   desc "remove", "Remove Traefik, app, and registry session from servers"
   def remove
     invoke "mrsk:cli:traefik:remove"

lib/mrsk/cli/prune.rb

@@ -1,5 +1,3 @@
-require "mrsk/cli/base"
-
 class Mrsk::Cli::Prune < Mrsk::Cli::Base
   desc "all", "Prune unused images and stopped containers"
   def all
@@ -9,11 +7,17 @@ class Mrsk::Cli::Prune < Mrsk::Cli::Base
 
   desc "images", "Prune unused images older than 30 days"
   def images
-    on(MRSK.hosts) { execute *MRSK.prune.images }
+    on(MRSK.hosts) do
+      execute *MRSK.auditor.record("prune images"), verbosity: :debug
+      execute *MRSK.prune.images
+    end
   end
 
   desc "containers", "Prune stopped containers for the service older than 3 days"
   def containers
-    on(MRSK.hosts) { execute *MRSK.prune.containers }
+    on(MRSK.hosts) do
+      execute *MRSK.auditor.record("prune containers"), verbosity: :debug
+      execute *MRSK.prune.containers
+    end
   end
 end

lib/mrsk/cli/registry.rb

@@ -1,9 +1,7 @@
-require "mrsk/cli/base"
-
 class Mrsk::Cli::Registry < Mrsk::Cli::Base
   desc "login", "Login to the registry locally and remotely"
   def login
-    run_locally           { execute *MRSK.registry.login }
+    run_locally    { execute *MRSK.registry.login }
     on(MRSK.hosts) { execute *MRSK.registry.login }
   rescue ArgumentError => e
     puts e.message

lib/mrsk/cli/server.rb

@@ -1,8 +1,6 @@
-require "mrsk/cli/base"
-
 class Mrsk::Cli::Server < Mrsk::Cli::Base
   desc "bootstrap", "Ensure Docker is installed on the servers"
   def bootstrap
-    on(MRSK.hosts) { execute "which docker || (apt-get update -y && apt-get install docker.io -y)" }
+    on(MRSK.hosts + MRSK.accessory_hosts) { execute "which docker || (apt-get update -y && apt-get install docker.io -y)" }
   end
 end

lib/mrsk/cli/templates/template.env

@@ -0,0 +1,2 @@
+MRSK_REGISTRY_PASSWORD=change-this
+RAILS_MASTER_KEY=another-env

lib/mrsk/cli/traefik.rb

@@ -1,5 +1,3 @@
-require "mrsk/cli/base"
-
 class Mrsk::Cli::Traefik < Mrsk::Cli::Base
   desc "boot", "Boot Traefik on servers"
   def boot
@@ -15,12 +13,18 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
 
   desc "start", "Start existing Traefik on servers"
   def start
-    on(MRSK.traefik_hosts) { execute *MRSK.traefik.start, raise_on_non_zero_exit: false }
+    on(MRSK.traefik_hosts) do
+      execute *MRSK.auditor.record("traefik start"), verbosity: :debug
+      execute *MRSK.traefik.start, raise_on_non_zero_exit: false
+    end
   end
 
   desc "stop", "Stop Traefik on servers"
   def stop
-    on(MRSK.traefik_hosts) { execute *MRSK.traefik.stop, raise_on_non_zero_exit: false }
+    on(MRSK.traefik_hosts) do
+      execute *MRSK.auditor.record("traefik stop"), verbosity: :debug
+      execute *MRSK.traefik.stop, raise_on_non_zero_exit: false
+    end
   end
 
   desc "restart", "Restart Traefik on servers"
@@ -50,7 +54,7 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
       end
     else
       since = options[:since]
-      lines = options[:lines]
+      lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
       on(MRSK.traefik_hosts) do |host|
         puts_by_host host, capture(*MRSK.traefik.logs(since: since, lines: lines, grep: grep)), type: "Traefik"
@@ -67,11 +71,17 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
 
   desc "remove_container", "Remove Traefik container from servers"
   def remove_container
-    on(MRSK.traefik_hosts) { execute *MRSK.traefik.remove_container }
+    on(MRSK.traefik_hosts) do
+      execute *MRSK.auditor.record("traefik remove container"), verbosity: :debug
+      execute *MRSK.traefik.remove_container
+    end
   end
 
   desc "remove_container", "Remove Traefik image from servers"
   def remove_image
-    on(MRSK.traefik_hosts) { execute *MRSK.traefik.remove_image }
+    on(MRSK.traefik_hosts) do
+      execute *MRSK.auditor.record("traefik remove image"), verbosity: :debug
+      execute *MRSK.traefik.remove_image
+    end
   end
 end

lib/mrsk/commander.rb

@@ -1,18 +1,10 @@
 require "active_support/core_ext/enumerable"
 
-require "mrsk/configuration"
-require "mrsk/commands/accessory"
-require "mrsk/commands/app"
-require "mrsk/commands/builder"
-require "mrsk/commands/prune"
-require "mrsk/commands/traefik"
-require "mrsk/commands/registry"
-
 class Mrsk::Commander
-  attr_accessor :config_file, :destination, :verbose, :version
+  attr_accessor :config_file, :destination, :verbosity, :version
 
-  def initialize(config_file: nil, destination: nil, verbose: false)
-    @config_file, @destination, @verbose = config_file, destination, verbose
+  def initialize(config_file: nil, destination: nil, verbosity: :info)
+    @config_file, @destination, @verbosity = config_file, destination, verbosity
   end
 
   def config
@@ -48,6 +40,10 @@ class Mrsk::Commander
     specific_hosts || config.accessories.collect(&:host)
   end
 
+  def accessory_names
+    config.accessories&.collect(&:name) || []
+  end
+
 
   def app
     @app ||= Mrsk::Commands::App.new(config)
@@ -70,11 +66,15 @@ class Mrsk::Commander
   end
 
   def accessory(name)
-    (@accessories ||= {})[name] ||= Mrsk::Commands::Accessory.new(config, name: name)
+    Mrsk::Commands::Accessory.new(config, name: name)
+  end
+
+  def auditor
+    @auditor ||= Mrsk::Commands::Auditor.new(config)
   end
 
 
-  def verbosity(level) 
+  def with_verbosity(level)
     old_level = SSHKit.config.output_verbosity
     SSHKit.config.output_verbosity = level
     yield
@@ -82,6 +82,13 @@ class Mrsk::Commander
     SSHKit.config.output_verbosity = old_level
   end
 
+  # Test-induced damage!
+  def reset
+    @config = @config_file = @destination = @version = nil
+    @app = @builder = @traefik = @registry = @prune = @auditor = nil
+    @verbosity = :info
+  end
+
   private
     def cascading_version
       version.presence || ENV["VERSION"] || `git rev-parse HEAD`.strip
@@ -91,6 +98,6 @@ class Mrsk::Commander
     def configure_sshkit_with(config)
       SSHKit::Backend::Netssh.configure { |ssh| ssh.ssh_options = config.ssh_options }
       SSHKit.config.command_map[:docker] = "docker" # No need to use /usr/bin/env, just clogs up the logs
-      SSHKit.config.output_verbosity = :debug if verbose
+      SSHKit.config.output_verbosity = verbosity
     end
 end

lib/mrsk/commands/accessory.rb

@@ -1,8 +1,6 @@
-require "mrsk/commands/base"
-
 class Mrsk::Commands::Accessory < Mrsk::Commands::Base
   attr_reader :accessory_config
-  delegate :service_name, :image, :host, :port, :env_args, :volume_args, :label_args, to: :accessory_config
+  delegate :service_name, :image, :host, :port, :files, :directories, :env_args, :volume_args, :label_args, to: :accessory_config
 
   def initialize(config, name:)
     super(config)
@@ -10,7 +8,7 @@ class Mrsk::Commands::Accessory < Mrsk::Commands::Base
   end
 
   def run
-    docker :run, 
+    docker :run,
       "--name", service_name,
       "-d",
       "--restart", "unless-stopped",
@@ -33,6 +31,7 @@ class Mrsk::Commands::Accessory < Mrsk::Commands::Base
     docker :ps, *service_filter
   end
 
+
   def logs(since: nil, lines: nil, grep: nil)
     pipe \
       docker(:logs, service_name, (" --since #{since}" if since), (" -n #{lines}" if lines), "-t", "2>&1"),
@@ -40,10 +39,59 @@ class Mrsk::Commands::Accessory < Mrsk::Commands::Base
   end
 
   def follow_logs(grep: nil)
-    run_over_ssh pipe(
-      docker(:logs, service_name, "-t", "-n", "10", "-f", "2>&1"),
-      ("grep '#{grep}'" if grep)
-    ).join(" "), host: host
+    run_over_ssh \
+      pipe \
+        docker(:logs, service_name, "-t", "-n", "10", "-f", "2>&1"),
+        (%(grep "#{grep}") if grep)
+  end
+
+
+  def execute_in_existing_container(*command, interactive: false)
+    docker :exec,
+      ("-it" if interactive),
+      service_name,
+      *command
+  end
+
+  def execute_in_new_container(*command, interactive: false)
+    docker :run,
+      ("-it" if interactive),
+      "--rm",
+      *env_args,
+      *volume_args,
+      image,
+      *command
+  end
+
+  def execute_in_existing_container_over_ssh(*command)
+    run_over_ssh execute_in_existing_container(*command, interactive: true)
+  end
+
+  def execute_in_new_container_over_ssh(*command)
+    run_over_ssh execute_in_new_container(*command, interactive: true)
+  end
+
+  def run_over_ssh(command)
+    super command, host: host
+  end
+
+
+  def ensure_local_file_present(local_file)
+    if !local_file.is_a?(StringIO) && !Pathname.new(local_file).exist?
+      raise "Missing file: #{local_file}"
+    end
+  end
+
+  def make_directory_for(remote_file)
+    make_directory Pathname.new(remote_file).dirname.to_s
+  end
+
+  def make_directory(path)
+    [ :mkdir, "-p", path ]
+  end
+
+  def remove_service_directory
+    [ :rm, "-rf", service_name ]
   end
 
   def remove_container

lib/mrsk/commands/app.rb

@@ -1,5 +1,3 @@
-require "mrsk/commands/base"
-
 class Mrsk::Commands::App < Mrsk::Commands::Base
   def run(role: :web)
     role = config.role(role)
@@ -7,8 +5,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
     docker :run,
       "-d",
       "--restart unless-stopped",
-      "--name", config.service_with_version,
-      *rails_master_key_arg,
+      "--name", service_with_version,
       *role.env_args,
       *config.volume_args,
       *role.label_args,
@@ -16,88 +13,119 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
       role.cmd
   end
 
-  def start(version: config.version)
-    docker :start, "#{config.service}-#{version}"
-  end
-
-  def current_container_id
-    docker :ps, "-q", *service_filter
+  def start
+    docker :start, service_with_version
   end
 
   def stop
-    pipe current_container_id, "xargs docker stop"
+    pipe current_container_id, xargs(docker(:stop))
   end
 
   def info
     docker :ps, *service_filter
   end
 
+
   def logs(since: nil, lines: nil, grep: nil)
     pipe \
       current_container_id,
-      "xargs docker logs#{" --since #{since}" if since}#{" -n #{lines}" if lines} -t 2>&1",
+      "xargs docker logs#{" --since #{since}" if since}#{" -n #{lines}" if lines} 2>&1",
       ("grep '#{grep}'" if grep)
   end
 
-  def exec(*command, interactive: false)
+  def follow_logs(host:, grep: nil)
+    run_over_ssh \
+      pipe(
+        current_container_id,
+        "xargs docker logs -t -n 10 -f 2>&1",
+        (%(grep "#{grep}") if grep)
+      ),
+      host: host
+  end
+
+
+  def execute_in_existing_container(*command, interactive: false)
     docker :exec,
       ("-it" if interactive),
-      *rails_master_key_arg,
-      *config.env_args,
-      *config.volume_args,
       config.service_with_version,
       *command
   end
 
-  def run_exec(*command, interactive: false)
+  def execute_in_new_container(*command, interactive: false)
     docker :run,
       ("-it" if interactive),
       "--rm",
-      *rails_master_key_arg,
       *config.env_args,
       *config.volume_args,
       config.absolute_image,
       *command
   end
 
-  def follow_logs(host:, grep: nil)
-    run_over_ssh pipe(
-      current_container_id,
-      "xargs docker logs -t -n 10 -f 2>&1",
-      ("grep '#{grep}'" if grep)
-    ).join(" "), host: host
+  def execute_in_existing_container_over_ssh(*command, host:)
+    run_over_ssh execute_in_existing_container(*command, interactive: true), host: host
   end
 
-  def console(host:)
-    exec_over_ssh "bin/rails", "c", host: host
+  def execute_in_new_container_over_ssh(*command, host:)
+    run_over_ssh execute_in_new_container(*command, interactive: true), host: host
   end
 
-  def bash(host:)
-    exec_over_ssh "bash", host: host
+
+  def current_container_id
+    docker :ps, "-q", *service_filter
   end
 
+  def container_id_for(container_name:)
+    docker :container, :ls, "-a", "-f", "name=#{container_name}", "-q"
+  end
+
+  def current_running_version
+    # FIXME: Find more graceful way to extract the version from "app-version" than using sed and tail!
+    pipe \
+      docker(:ps, "--filter", "label=service=#{config.service}", "--format", '"{{.Names}}"'),
+      %(sed 's/-/\\n/g'),
+      "tail -n 1"
+  end
+
+  def most_recent_version_from_available_images
+    pipe \
+      docker(:image, :ls, "--format", '"{{.Tag}}"', config.repository),
+      "head -n 1"
+  end
+
+
   def list_containers
     docker :container, :ls, "-a", *service_filter
   end
 
+  def remove_container(version:)
+    pipe \
+      container_id_for(container_name: service_with_version(version)),
+      xargs(docker(:container, :rm))
+  end
+
   def remove_containers
     docker :container, :prune, "-f", *service_filter
   end
 
+  def list_images
+    docker :image, :ls, config.repository
+  end
+
   def remove_images
     docker :image, :prune, "-a", "-f", *service_filter
   end
 
+
   private
-    def exec_over_ssh(*command, host:)
-      run_over_ssh run_exec(*command, interactive: true).join(" "), host: host
+    def service_with_version(version = nil)
+      if version
+        "#{config.service}-#{version}"
+      else
+        config.service_with_version
+      end
     end
 
     def service_filter
       [ "--filter", "label=service=#{config.service}" ]
     end
-
-    def rails_master_key_arg
-      [ "-e", redact("RAILS_MASTER_KEY=#{config.master_key}") ]
-    end
 end

lib/mrsk/commands/auditor.rb

@@ -0,0 +1,34 @@
+require "active_support/core_ext/time/conversions"
+
+class Mrsk::Commands::Auditor < Mrsk::Commands::Base
+  def record(line)
+    append \
+      [ :echo, tagged_line(line) ],
+      audit_log_file
+  end
+
+  def reveal
+    [ :tail, "-n", 50, audit_log_file ]
+  end
+
+  private
+    def audit_log_file
+      "mrsk-#{config.service}-audit.log"
+    end
+
+    def tagged_line(line)
+      "'#{tags} #{line}'"
+    end
+
+    def tags
+      "[#{timestamp}] [#{performer}]"
+    end
+
+    def performer
+      `whoami`.strip
+    end
+
+    def timestamp
+      Time.now.to_fs(:db)
+    end
+end

lib/mrsk/commands/base.rb

@@ -8,6 +8,13 @@ module Mrsk::Commands
       @config = config
     end
 
+    def run_over_ssh(*command, host:)
+      "ssh".tap do |cmd|
+        cmd << " -J #{config.ssh_proxy.jump_proxies}" if config.ssh_proxy
+        cmd << " -t #{config.ssh_user}@#{host} '#{command.join(" ")}'"
+      end
+    end
+
     private
       def combine(*commands, by: "&&")
         commands
@@ -16,16 +23,24 @@ module Mrsk::Commands
           .tap     { |commands| commands.pop } # Remove trailing combiner
       end
 
+      def chain(*commands)
+        combine *commands, by: ";"
+      end
+
       def pipe(*commands)
         combine *commands, by: "|"
       end
 
+      def append(*commands)
+        combine *commands, by: ">>"
+      end
+
+      def xargs(command)
+        [ :xargs, command ].flatten
+      end
+
       def docker(*args)
         args.compact.unshift :docker
       end
-
-      def run_over_ssh(command, host:)
-        "ssh -t #{config.ssh_user}@#{host} '#{command}'"
-      end
   end
 end

lib/mrsk/commands/builder.rb

@@ -1,5 +1,3 @@
-require "mrsk/commands/base"
-
 class Mrsk::Commands::Builder < Mrsk::Commands::Base
   delegate :create, :remove, :push, :pull, :info, to: :target
 
@@ -36,8 +34,3 @@ class Mrsk::Commands::Builder < Mrsk::Commands::Base
     @multiarch_remote ||= Mrsk::Commands::Builder::Multiarch::Remote.new(config)
   end
 end
-
-require "mrsk/commands/builder/native"
-require "mrsk/commands/builder/native/remote"
-require "mrsk/commands/builder/multiarch"
-require "mrsk/commands/builder/multiarch/remote"

lib/mrsk/commands/builder/base.rb

@@ -1,5 +1,3 @@
-require "mrsk/commands/base"
-
 class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
   delegate :argumentize, to: Mrsk::Utils
 

lib/mrsk/commands/builder/multiarch.rb

@@ -1,5 +1,3 @@
-require "mrsk/commands/builder/base"
-
 class Mrsk::Commands::Builder::Multiarch < Mrsk::Commands::Builder::Base
   def create
     docker :buildx, :create, "--use", "--name", builder_name

lib/mrsk/commands/builder/multiarch/remote.rb

@@ -1,5 +1,3 @@
-require "mrsk/commands/builder/multiarch"
-
 class Mrsk::Commands::Builder::Multiarch::Remote < Mrsk::Commands::Builder::Multiarch
   def create
     combine \

lib/mrsk/commands/builder/native.rb

@@ -1,5 +1,3 @@
-require "mrsk/commands/builder/base"
-
 class Mrsk::Commands::Builder::Native < Mrsk::Commands::Builder::Base
   def create
     # No-op on native

lib/mrsk/commands/builder/native/remote.rb

@@ -1,14 +1,12 @@
-require "mrsk/commands/builder/native"
-
 class Mrsk::Commands::Builder::Native::Remote < Mrsk::Commands::Builder::Native
   def create
-    combine \
+    chain \
       create_context,
       create_buildx
   end
 
   def remove
-    combine \
+    chain \
       remove_context,
       remove_buildx
   end
@@ -25,7 +23,7 @@ class Mrsk::Commands::Builder::Native::Remote < Mrsk::Commands::Builder::Native
   end
 
   def info
-    combine \
+    chain \
       docker(:context, :ls),
       docker(:buildx, :ls)
   end

lib/mrsk/commands/prune.rb

@@ -1,4 +1,3 @@
-require "mrsk/commands/base"
 require "active_support/duration"
 require "active_support/core_ext/numeric/time"
 

lib/mrsk/commands/registry.rb

@@ -1,5 +1,3 @@
-require "mrsk/commands/base"
-
 class Mrsk::Commands::Registry < Mrsk::Commands::Base
   delegate :registry, to: :config
 

lib/mrsk/commands/traefik.rb

@@ -1,5 +1,3 @@
-require "mrsk/commands/base"
-
 class Mrsk::Commands::Traefik < Mrsk::Commands::Base
   def run
     docker :run, "--name traefik",
@@ -9,7 +7,8 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
       "-v /var/run/docker.sock:/var/run/docker.sock",
       "traefik",
       "--providers.docker",
-      "--log.level=DEBUG"
+      "--log.level=DEBUG",
+      *cmd_args
   end
 
   def start
@@ -33,7 +32,7 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
   def follow_logs(host:, grep: nil)
     run_over_ssh pipe(
       docker(:logs, "traefik", "-t", "-n", "10", "-f", "2>&1"),
-      ("grep '#{grep}'" if grep)
+      (%(grep "#{grep}") if grep)
     ).join(" "), host: host
   end
 
@@ -44,4 +43,9 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
   def remove_image
     docker :image, :prune, "-a", "-f", "--filter", "label=org.opencontainers.image.title=Traefik"
   end
+
+  private
+    def cmd_args
+      (config.raw_config.dig(:traefik, "args") || { }).collect { |(key, value)| [ "--#{key}", value ] }.flatten
+    end
 end

lib/mrsk/configuration.rb

@@ -3,19 +3,20 @@ require "active_support/core_ext/string/inquiry"
 require "active_support/core_ext/module/delegation"
 require "pathname"
 require "erb"
-require "mrsk/utils"
+require "net/ssh/proxy/jump"
 
 class Mrsk::Configuration
   delegate :service, :image, :servers, :env, :labels, :registry, :builder, to: :raw_config, allow_nil: true
   delegate :argumentize, :argumentize_env_with_secrets, to: Mrsk::Utils
 
+  attr_accessor :version
   attr_accessor :raw_config
 
   class << self
     def create_from(base_config_file, destination: nil, version: "missing")
       new(load_config_file(base_config_file).tap do |config|
         if destination
-          config.merge! \
+          config.deep_merge! \
             load_config_file destination_config_file(base_config_file, destination)
         end
       end, version: version)
@@ -39,7 +40,7 @@ class Mrsk::Configuration
   def initialize(raw_config, version: "missing", validate: true)
     @raw_config = ActiveSupport::InheritableOptions.new(raw_config)
     @version = version
-    ensure_required_keys_present if validate
+    valid? if validate
   end
 
 
@@ -52,7 +53,7 @@ class Mrsk::Configuration
   end
 
   def accessories
-    @accessories ||= raw_config.accessories.keys.collect { |name| Mrsk::Configuration::Assessory.new(name, config: self) }
+    @accessories ||= raw_config.accessories&.keys&.collect { |name| Mrsk::Configuration::Accessory.new(name, config: self) } || []
   end
 
   def accessory(name)
@@ -73,10 +74,6 @@ class Mrsk::Configuration
   end
 
 
-  def version
-    @version
-  end
-
   def repository
     [ raw_config.registry["server"], image ].compact.join("/")
   end
@@ -107,17 +104,30 @@ class Mrsk::Configuration
   end
 
   def ssh_user
-    raw_config.ssh_user || "root"
+    if raw_config.ssh.present?
+      raw_config.ssh["user"] || "root"
+    else
+      "root"
+    end
+  end
+
+  def ssh_proxy
+    if raw_config.ssh.present? && raw_config.ssh["proxy"]
+      Net::SSH::Proxy::Jump.new \
+        raw_config.ssh["proxy"].include?("@") ? raw_config.ssh["proxy"] : "root@#{raw_config.ssh["proxy"]}"
+    end
   end
 
   def ssh_options
-    { user: ssh_user, auth_methods: [ "publickey" ] }
+    { user: ssh_user, proxy: ssh_proxy, auth_methods: [ "publickey" ] }.compact
   end
 
-  def master_key
-    ENV["RAILS_MASTER_KEY"] || File.read(Pathname.new(File.expand_path("config/master.key")))
+
+  def valid?
+    ensure_required_keys_present && ensure_env_available
   end
 
+
   def to_h
     {
       roles: role_names,
@@ -130,12 +140,14 @@ class Mrsk::Configuration
       env_args: env_args,
       volume_args: volume_args,
       ssh_options: ssh_options,
-      builder: raw_config.builder
+      builder: raw_config.builder,
+      accessories: raw_config.accessories
     }.compact
   end
 
 
   private
+    # Will raise ArgumentError if any required config keys are missing
     def ensure_required_keys_present
       %i[ service image registry servers ].each do |key|
         raise ArgumentError, "Missing required configuration for #{key}" unless raw_config[key].present?
@@ -148,12 +160,19 @@ class Mrsk::Configuration
       if raw_config.registry["password"].blank?
         raise ArgumentError, "You must specify a password for the registry in config/deploy.yml (or set the ENV variable if that's used)"
       end
+
+      true
+    end
+
+    # Will raise KeyError if any secret ENVs are missing
+    def ensure_env_available
+      env_args
+      roles.each(&:env_args)
+
+      true
     end
 
     def role_names
       raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort
     end
 end
-
-require "mrsk/configuration/role"
-require "mrsk/configuration/accessory"

lib/mrsk/configuration/accessory.rb

@@ -1,4 +1,4 @@
-class Mrsk::Configuration::Assessory
+class Mrsk::Configuration::Accessory
   delegate :argumentize, :argumentize_env_with_secrets, to: Mrsk::Utils
 
   attr_accessor :name, :specifics
@@ -43,8 +43,22 @@ class Mrsk::Configuration::Assessory
     argumentize_env_with_secrets env
   end
 
+  def files
+    specifics["files"]&.to_h do |local_to_remote_mapping|
+      local_file, remote_file = local_to_remote_mapping.split(":")
+      [ expand_local_file(local_file), expand_remote_file(remote_file) ]
+    end || {}
+  end
+
+  def directories
+    specifics["directories"]&.to_h do |host_to_container_mapping|
+      host_relative_path, container_path = host_to_container_mapping.split(":")
+      [ expand_host_path(host_relative_path), container_path ]
+    end || {}
+  end
+
   def volumes
-    specifics["volumes"] || []
+    specific_volumes + remote_files_as_volumes + remote_directories_as_volumes
   end
 
   def volume_args
@@ -57,4 +71,53 @@ class Mrsk::Configuration::Assessory
     def default_labels
       { "service" => service_name }
     end
+
+    def expand_local_file(local_file)
+      if local_file.end_with?("erb")
+        with_clear_env_loaded { read_dynamic_file(local_file) }
+      else
+        Pathname.new(File.expand_path(local_file)).to_s
+      end
+    end
+
+    def with_clear_env_loaded
+      (env["clear"] || env).each { |k, v| ENV[k] = v }
+      yield
+    ensure
+      (env["clear"] || env).each { |k, v| ENV.delete(k) }
+    end
+
+    def read_dynamic_file(local_file)
+      StringIO.new(ERB.new(IO.read(local_file)).result)
+    end
+
+    def expand_remote_file(remote_file)
+      service_name + remote_file
+    end
+
+    def specific_volumes
+      specifics["volumes"] || []
+    end
+
+    def remote_files_as_volumes
+      specifics["files"]&.collect do |local_to_remote_mapping|
+        _, remote_file = local_to_remote_mapping.split(":")
+        "#{service_data_directory + remote_file}:#{remote_file}"
+      end || []
+    end
+
+    def remote_directories_as_volumes
+      specifics["directories"]&.collect do |host_to_container_mapping|
+        host_relative_path, container_path = host_to_container_mapping.split(":")
+        [ expand_host_path(host_relative_path), container_path ].join(":")
+      end || []
+    end
+
+    def expand_host_path(host_relative_path)
+      "#{service_data_directory}/#{host_relative_path}"
+    end
+
+    def service_data_directory
+      "$PWD/#{service_name}"
+    end
 end

lib/mrsk/configuration/role.rb

@@ -96,7 +96,12 @@ class Mrsk::Configuration::Role
     def merged_env_with_secrets
       merged_env.tap do |new_env|
         new_env["secret"] = Array(config.env["secret"]) + Array(specialized_env["secret"])
-        new_env["clear"]  = (Array(config.env["clear"] || config.env) + Array(specialized_env["clear"] || specialized_env)).uniq
+
+        # If there's no secret/clear split, everything is clear
+        clear_app_env  = config.env["secret"] ? Array(config.env["clear"]) : Array(config.env["clear"] || config.env)
+        clear_role_env = specialized_env["secret"] ? Array(specialized_env["clear"]) : Array(specialized_env["clear"] || specialized_env)
+
+        new_env["clear"] = (clear_app_env + clear_role_env).uniq
       end
     end
 end

lib/mrsk/utils.rb

@@ -18,7 +18,7 @@ module Mrsk::Utils
     if (secrets = env["secret"]).present?
       argumentize("-e", secrets.to_h { |key| [ key, ENV.fetch(key) ] }, redacted: true) + argumentize("-e", env["clear"])
     else
-      argumentize "-e", env
+      argumentize "-e", env.fetch("clear", env)
     end
   end
 

lib/mrsk/version.rb

@@ -1,3 +1,3 @@
 module Mrsk
-  VERSION = "0.2.0"
+  VERSION = "0.6.0"
 end

mrsk.gemspec

@@ -15,4 +15,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "activesupport", ">= 7.0"
   spec.add_dependency "sshkit", "~> 1.21"
   spec.add_dependency "thor", "~> 1.2"
+  spec.add_dependency "dotenv", "~> 2.8"
+  spec.add_dependency "zeitwerk", "~> 2.5"
 end

test/cli/accessory_test.rb

@@ -1,17 +1,38 @@
-require "test_helper"
-require "active_support/testing/stream"
-require "mrsk/cli"
+require_relative "cli_test_case"
 
-class CliAccessoryTest < ActiveSupport::TestCase
-  include ActiveSupport::Testing::Stream
+class CliAccessoryTest < CliTestCase
+  test "upload" do
+    assert_match "test/fixtures/files/my.cnf app-mysql/etc/mysql/my.cnf", run_command("upload", "mysql")
+  end
+
+  test "directories" do
+    assert_match "mkdir -p $PWD/app-mysql/data", run_command("directories", "mysql")
+  end
+
+  test "remove service direcotry" do
+    assert_match "rm -rf app-mysql", run_command("remove_service_directory", "mysql")
+  end
 
   test "boot" do
-    ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
-
-    command = stdouted { Mrsk::Cli::Accessory.start(["boot", "mysql", "-c", "test/fixtures/deploy_with_accessories.yml"]) }
-    
-    assert_match "Running docker run --name app-mysql -d --restart unless-stopped -p 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=% --volume /var/lib/mysql:/var/lib/mysql --label service=app-mysql mysql:5.7 on 1.1.1.3", command
-  ensure
-    ENV["MYSQL_ROOT_PASSWORD"] = nil
+    assert_match "Running docker run --name app-mysql -d --restart unless-stopped -p 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=% --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=app-mysql mysql:5.7 on 1.1.1.3", run_command("boot", "mysql")
   end
+
+  test "exec" do
+    run_command("exec", "mysql", "mysql -v").tap do |output|
+      assert_match /Launching command from new container/, output
+      assert_match /mysql -v/, output
+    end
+  end
+
+  test "exec with reuse" do
+    run_command("exec", "mysql", "--reuse", "mysql -v").tap do |output|
+      assert_match /Launching command from existing container/, output
+      assert_match %r[docker exec app-mysql mysql -v], output
+    end
+  end
+
+  private
+    def run_command(*command)
+      stdouted { Mrsk::Cli::Accessory.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+    end
 end

test/cli/app_test.rb

@@ -0,0 +1,67 @@
+require_relative "cli_test_case"
+
+class CliAppTest < CliTestCase
+  test "boot" do
+    assert_match /Running docker run -d --restart unless-stopped/, run_command("boot")
+  end
+
+  test "boot will reboot if same version is already running" do
+    run_command("details") # Preheat MRSK const
+
+    # Prevent expected failures from outputting to terminal
+    Thread.report_on_exception = false
+
+    MRSK.app.stubs(:run).raises(SSHKit::Command::Failed.new("already in use")).then.returns([ :docker, :run ])
+
+    run_command("boot").tap do |output|
+      assert_match /Rebooting container with same version already deployed/, output # Can't start what's already running
+      assert_match /docker ps -q --filter label=service=app \| xargs docker stop/, output # Stop what's running
+      assert_match /docker container ls -a -f name=app-999 -q \| xargs docker container rm/, output # Remove old container
+      assert_match /docker run/, output # Start new container
+    end
+  ensure
+    Thread.report_on_exception = true
+  end
+
+  test "start" do
+    run_command("start").tap do |output|
+      assert_match /docker start app-999/, output
+    end
+  end
+
+  test "stop" do
+    run_command("stop").tap do |output|
+      assert_match /docker ps -q --filter label=service=app \| xargs docker stop/, output
+    end
+  end
+
+  test "details" do
+    run_command("details").tap do |output|
+      assert_match /docker ps --filter label=service=app/, output
+    end
+  end
+
+  test "remove_container" do
+    run_command("remove_container", "1234567").tap do |output|
+      assert_match /docker container ls -a -f name=app-1234567 -q \| xargs docker container rm/, output
+    end
+  end
+
+  test "exec" do
+    run_command("exec", "ruby -v").tap do |output|
+      assert_match /ruby -v/, output
+    end
+  end
+
+  test "exec with reuse" do
+    run_command("exec", "--reuse", "ruby -v").tap do |output|
+      assert_match %r[docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1], output # Get current version
+      assert_match %r[docker exec app-999 ruby -v], output
+    end
+  end
+
+  private
+    def run_command(*command)
+      stdouted { Mrsk::Cli::App.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+    end
+end

test/cli/cli_test_case.rb

@@ -0,0 +1,24 @@
+require "test_helper"
+require "active_support/testing/stream"
+
+class CliTestCase < ActiveSupport::TestCase
+  include ActiveSupport::Testing::Stream
+
+  setup do
+    ENV["VERSION"]             = "999"
+    ENV["RAILS_MASTER_KEY"]    = "123"
+    ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
+  end
+
+  teardown do
+    ENV.delete("RAILS_MASTER_KEY")
+    ENV.delete("MYSQL_ROOT_PASSWORD")
+    ENV.delete("VERSION")
+    MRSK.reset
+  end
+
+  private
+    def stdouted
+      capture(:stdout) { yield }.strip
+    end
+end

test/cli/main_test.rb

@@ -1,13 +1,6 @@
-require "test_helper"
-require "active_support/testing/stream"
-require "mrsk/cli"
-
-class CliMainTest < ActiveSupport::TestCase
-  include ActiveSupport::Testing::Stream
-
-  setup do
-  end
+require_relative "cli_test_case"
 
+class CliMainTest < CliTestCase
   test "version" do
     version = stdouted { Mrsk::Cli::Main.new.version }
     assert_equal Mrsk::VERSION, version

test/commander_test.rb

@@ -1,5 +1,4 @@
 require "test_helper"
-require "mrsk/commander"
 
 class CommanderTest < ActiveSupport::TestCase
   setup do

test/commands/accessory_test.rb

@@ -1,10 +1,8 @@
 require "test_helper"
-require "mrsk/configuration"
-require "mrsk/commands/accessory"
 
 class CommandsAccessoryTest < ActiveSupport::TestCase
   setup do
-    @config = { 
+    @config = {
       service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
       servers: [ "1.1.1.1" ],
       accessories: {
@@ -41,18 +39,20 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
     @config = Mrsk::Configuration.new(@config)
     @mysql  = Mrsk::Commands::Accessory.new(@config, name: :mysql)
     @redis  = Mrsk::Commands::Accessory.new(@config, name: :redis)
+
+    ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
+  end
+
+  teardown do
+    ENV.delete("MYSQL_ROOT_PASSWORD")
   end
 
   test "run" do
-    ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
-
     assert_equal \
       [:docker, :run, "--name", "app-mysql", "-d", "--restart", "unless-stopped", "-p", "3306:3306", "-e", "MYSQL_ROOT_PASSWORD=secret123", "-e", "MYSQL_ROOT_HOST=%", "--label", "service=app-mysql", "mysql:8.0"], @mysql.run
 
     assert_equal \
       [:docker, :run, "--name", "app-redis", "-d", "--restart", "unless-stopped", "-p", "6379:6379", "-e", "SOMETHING=else", "--volume", "/var/lib/redis:/data", "--label", "service=app-redis", "--label", "cache=true", "redis:latest"], @redis.run
-  ensure
-    ENV["MYSQL_ROOT_PASSWORD"] = nil
   end
 
   test "start" do
@@ -67,6 +67,35 @@ class CommandsAccessoryTest < ActiveSupport::TestCase
     assert_equal [:docker, :ps, "--filter", "label=service=app-mysql"], @mysql.info
   end
 
+
+  test "execute in new container" do
+    assert_equal \
+      [ :docker, :run, "--rm", "-e", "MYSQL_ROOT_PASSWORD=secret123", "-e", "MYSQL_ROOT_HOST=%", "mysql:8.0", "mysql", "-u", "root" ],
+      @mysql.execute_in_new_container("mysql", "-u", "root")
+  end
+
+  test "execute in existing container" do
+    assert_equal \
+      [ :docker, :exec, "app-mysql", "mysql", "-u", "root" ],
+      @mysql.execute_in_existing_container("mysql", "-u", "root")
+  end
+
+  test "execute in new container over ssh" do
+    @mysql.stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do
+      assert_match %r|docker run -it --rm -e MYSQL_ROOT_PASSWORD=secret123 -e MYSQL_ROOT_HOST=% mysql:8.0 mysql -u root|,
+        @mysql.execute_in_new_container_over_ssh("mysql", "-u", "root")
+    end
+  end
+
+  test "execute in existing container over ssh" do
+    @mysql.stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do
+      assert_match %r|docker exec -it app-mysql mysql -u root|,
+        @mysql.execute_in_existing_container_over_ssh("mysql", "-u", "root")
+    end
+  end
+
+
+
   test "logs" do
     assert_equal [:docker, :logs, "app-mysql", "-t", "2>&1"], @mysql.logs
     assert_equal [:docker, :logs, "app-mysql", " --since 5m", " -n 100", "-t", "2>&1", "|", "grep 'thing'"], @mysql.logs(since: "5m", lines: 100, grep: "thing")

test/commands/app_test.rb

@@ -1,30 +1,161 @@
 require "test_helper"
-require "mrsk/configuration"
-require "mrsk/commands/app"
-
-ENV["RAILS_MASTER_KEY"] = "456"
 
 class CommandsAppTest < ActiveSupport::TestCase
   setup do
-    @config = { service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ] }
-    @app = Mrsk::Commands::App.new Mrsk::Configuration.new(@config)
+    ENV["RAILS_MASTER_KEY"] = "456"
+
+    @config = { service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], env: { "secret" => [ "RAILS_MASTER_KEY" ] } }
+    @app = Mrsk::Commands::App.new Mrsk::Configuration.new(@config).tap { |c| c.version = "999" }
+  end
+
+  teardown do
+    ENV.delete("RAILS_MASTER_KEY")
   end
 
   test "run" do
     assert_equal \
-      [:docker, :run, "-d", "--restart unless-stopped", "--name", "app-missing", "-e", "RAILS_MASTER_KEY=456", "--label", "service=app", "--label", "role=web", "--label", "traefik.http.routers.app.rule='PathPrefix(`/`)'", "--label", "traefik.http.services.app.loadbalancer.healthcheck.path=/up", "--label", "traefik.http.services.app.loadbalancer.healthcheck.interval=1s", "--label", "traefik.http.middlewares.app.retry.attempts=3", "--label", "traefik.http.middlewares.app.retry.initialinterval=500ms", "dhh/app:missing"], @app.run
+      "docker run -d --restart unless-stopped --name app-999 -e RAILS_MASTER_KEY=456 --label service=app --label role=web --label traefik.http.routers.app.rule='PathPrefix(`/`)' --label traefik.http.services.app.loadbalancer.healthcheck.path=/up --label traefik.http.services.app.loadbalancer.healthcheck.interval=1s --label traefik.http.middlewares.app.retry.attempts=3 --label traefik.http.middlewares.app.retry.initialinterval=500ms dhh/app:999",
+      @app.run.join(" ")
   end
 
   test "run with volumes" do
     @config[:volumes] = ["/local/path:/container/path" ]
 
     assert_equal \
-      [:docker, :run, "-d", "--restart unless-stopped", "--name", "app-missing", "-e", "RAILS_MASTER_KEY=456", "--volume", "/local/path:/container/path", "--label", "service=app", "--label", "role=web", "--label", "traefik.http.routers.app.rule='PathPrefix(`/`)'", "--label", "traefik.http.services.app.loadbalancer.healthcheck.path=/up", "--label", "traefik.http.services.app.loadbalancer.healthcheck.interval=1s", "--label", "traefik.http.middlewares.app.retry.attempts=3", "--label", "traefik.http.middlewares.app.retry.initialinterval=500ms", "dhh/app:missing"], @app.run
+      "docker run -d --restart unless-stopped --name app-999 -e RAILS_MASTER_KEY=456 --volume /local/path:/container/path --label service=app --label role=web --label traefik.http.routers.app.rule='PathPrefix(`/`)' --label traefik.http.services.app.loadbalancer.healthcheck.path=/up --label traefik.http.services.app.loadbalancer.healthcheck.interval=1s --label traefik.http.middlewares.app.retry.attempts=3 --label traefik.http.middlewares.app.retry.initialinterval=500ms dhh/app:999",
+      @app.run.join(" ")
   end
 
-  test "run with" do
+  test "start" do
     assert_equal \
-      [ :docker, :run, "--rm", "-e", "RAILS_MASTER_KEY=456", "dhh/app:missing", "bin/rails", "db:setup" ],
-      @app.run_exec("bin/rails", "db:setup")
+      "docker start app-999",
+      @app.start.join(" ")
+  end
+
+  test "stop" do
+    assert_equal \
+      "docker ps -q --filter label=service=app | xargs docker stop",
+      @app.stop.join(" ")
+  end
+
+  test "info" do
+    assert_equal \
+      "docker ps --filter label=service=app",
+      @app.info.join(" ")
+  end
+
+
+  test "logs" do
+    assert_equal \
+      "docker ps -q --filter label=service=app | xargs docker logs 2>&1",
+      @app.logs.join(" ")
+
+    assert_equal \
+      "docker ps -q --filter label=service=app | xargs docker logs --since 5m 2>&1",
+      @app.logs(since: "5m").join(" ")
+
+    assert_equal \
+      "docker ps -q --filter label=service=app | xargs docker logs -n 100 2>&1",
+      @app.logs(lines: "100").join(" ")
+
+    assert_equal \
+      "docker ps -q --filter label=service=app | xargs docker logs --since 5m -n 100 2>&1",
+      @app.logs(since: "5m", lines: "100").join(" ")
+
+    assert_equal \
+      "docker ps -q --filter label=service=app | xargs docker logs 2>&1 | grep 'my-id'",
+      @app.logs(grep: "my-id").join(" ")
+
+    assert_equal \
+      "docker ps -q --filter label=service=app | xargs docker logs --since 5m 2>&1 | grep 'my-id'",
+      @app.logs(since: "5m", grep: "my-id").join(" ")
+  end
+
+  test "follow logs" do
+    @app.stub(:run_over_ssh, ->(cmd, host:) { cmd.join(" ") }) do
+      assert_equal \
+        "docker ps -q --filter label=service=app | xargs docker logs -t -n 10 -f 2>&1",
+        @app.follow_logs(host: "app-1")
+
+      assert_equal \
+        "docker ps -q --filter label=service=app | xargs docker logs -t -n 10 -f 2>&1 | grep \"Completed\"",
+        @app.follow_logs(host: "app-1", grep: "Completed")
+    end
+  end
+
+
+  test "execute in new container" do
+    assert_equal \
+      "docker run --rm -e RAILS_MASTER_KEY=456 dhh/app:999 bin/rails db:setup",
+      @app.execute_in_new_container("bin/rails", "db:setup").join(" ")
+  end
+
+  test "execute in existing container" do
+    assert_equal \
+      "docker exec app-999 bin/rails db:setup",
+      @app.execute_in_existing_container("bin/rails", "db:setup").join(" ")
+  end
+
+  test "execute in new container over ssh" do
+    @app.stub(:run_over_ssh, ->(cmd, host:) { cmd.join(" ") }) do
+      assert_match %r|docker run -it --rm -e RAILS_MASTER_KEY=456 dhh/app:999 bin/rails c|,
+        @app.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1")
+    end
+  end
+
+  test "execute in existing container over ssh" do
+    @app.stub(:run_over_ssh, ->(cmd, host:) { cmd.join(" ") }) do
+      assert_match %r|docker exec -it app-999 bin/rails c|,
+        @app.execute_in_existing_container_over_ssh("bin/rails", "c", host: "app-1")
+    end
+  end
+
+  test "run over ssh" do
+    assert_equal "ssh -t root@1.1.1.1 'ls'", @app.run_over_ssh("ls", host: "1.1.1.1")
+  end
+
+  test "run over ssh with custom user" do
+    @app = Mrsk::Commands::App.new Mrsk::Configuration.new(@config.tap { |c| c[:ssh] = { "user" => "app" } })
+    assert_equal "ssh -t app@1.1.1.1 'ls'", @app.run_over_ssh("ls", host: "1.1.1.1")
+  end
+
+  test "run over ssh with proxy" do
+    @app = Mrsk::Commands::App.new Mrsk::Configuration.new(@config.tap { |c| c[:ssh] = { "proxy" => "2.2.2.2" } })
+    assert_equal "ssh -J root@2.2.2.2 -t root@1.1.1.1 'ls'", @app.run_over_ssh("ls", host: "1.1.1.1")
+  end
+
+  test "run over ssh with proxy user" do
+    @app = Mrsk::Commands::App.new Mrsk::Configuration.new(@config.tap { |c| c[:ssh] = { "proxy" => "app@2.2.2.2" } })
+    assert_equal "ssh -J app@2.2.2.2 -t root@1.1.1.1 'ls'", @app.run_over_ssh("ls", host: "1.1.1.1")
+  end
+
+  test "run over ssh with custom user with proxy" do
+    @app = Mrsk::Commands::App.new Mrsk::Configuration.new(@config.tap { |c| c[:ssh] = { "user" => "app", "proxy" => "2.2.2.2" } })
+    assert_equal "ssh -J root@2.2.2.2 -t app@1.1.1.1 'ls'", @app.run_over_ssh("ls", host: "1.1.1.1")
+  end
+
+
+  test "current_container_id" do
+    assert_equal \
+      "docker ps -q --filter label=service=app",
+      @app.current_container_id.join(" ")
+  end
+
+  test "container_id_for" do
+    assert_equal \
+      "docker container ls -a -f name=app-999 -q",
+      @app.container_id_for(container_name: "app-999").join(" ")
+  end
+
+  test "current_running_version" do
+    assert_equal \
+      "docker ps --filter label=service=app --format \"{{.Names}}\" | sed 's/-/\\n/g' | tail -n 1",
+      @app.current_running_version.join(" ")
+  end
+
+  test "most_recent_version_from_available_images" do
+    assert_equal \
+      "docker image ls --format \"{{.Tag}}\" dhh/app | head -n 1",
+      @app.most_recent_version_from_available_images.join(" ")
   end
 end

test/commands/builder_test.rb

@@ -1,6 +1,4 @@
 require "test_helper"
-require "mrsk/configuration"
-require "mrsk/commands/builder"
 
 class CommandsBuilderTest < ActiveSupport::TestCase
   setup do

test/commands/registry_test.rb

@@ -0,0 +1,23 @@
+require "test_helper"
+
+class CommandsRegistryTest < ActiveSupport::TestCase
+  setup do
+    @config = { service: "app",
+      image: "dhh/app",
+      registry: { "username" => "dhh",
+        "password" => "secret",
+        "server" => "hub.docker.com"
+      },
+      servers: [ "1.1.1.1" ]
+    }
+    @registry = Mrsk::Commands::Registry.new Mrsk::Configuration.new(@config)
+  end
+
+  test "registry login" do
+    assert_equal [ :docker, :login, "hub.docker.com", "-u", "dhh", "-p", "secret" ], @registry.login
+  end
+
+  test "registry logout" do
+    assert_equal [:docker, :logout, "hub.docker.com"], @registry.logout
+  end
+end

test/commands/traefik_test.rb

@@ -0,0 +1,76 @@
+require "test_helper"
+
+class CommandsTraefikTest < ActiveSupport::TestCase
+  setup do
+    @config = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ],
+      traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
+    }
+  end
+
+  test "run" do
+    assert_equal \
+      [:docker, :run, "--name traefik", "-d", "--restart unless-stopped", "-p 80:80", "-v /var/run/docker.sock:/var/run/docker.sock", "traefik", "--providers.docker", "--log.level=DEBUG", "--accesslog.format", "json", "--metrics.prometheus.buckets", "0.1,0.3,1.2,5.0"],
+      new_command.run
+  end
+
+  test "traefik start" do
+    assert_equal \
+      [:docker, :container, :start, 'traefik'], new_command.start
+  end
+
+  test "traefik stop" do
+    assert_equal \
+      [:docker, :container, :stop, 'traefik'], new_command.stop
+  end
+
+  test "traefik info" do
+    assert_equal \
+      [:docker, :ps, '--filter', 'name=traefik'], new_command.info
+  end
+
+  test "traefik logs" do
+    assert_equal \
+      [:docker, :logs, 'traefik', '-t', '2>&1'], new_command.logs
+  end
+
+  test "traefik logs since 2h" do
+    assert_equal \
+      [:docker, :logs, 'traefik', ' --since 2h', '-t', '2>&1'], new_command.logs(since: '2h')
+  end
+
+  test "traefik logs last 10 lines" do
+    assert_equal \
+      [:docker, :logs, 'traefik', ' -n 10', '-t', '2>&1'], new_command.logs(lines: 10)
+  end
+
+  test "traefik logs with grep hello!" do
+    assert_equal \
+      [:docker, :logs, 'traefik', '-t', '2>&1', "|", "grep 'hello!'"], new_command.logs(grep: 'hello!')
+  end
+
+  test "traefik remove container" do
+    assert_equal \
+      [:docker, :container, :prune, "-f", "--filter", "label=org.opencontainers.image.title=Traefik"], new_command.remove_container
+  end
+
+  test "traefik remove image" do
+    assert_equal \
+    [:docker, :image, :prune, "-a", "-f", "--filter", "label=org.opencontainers.image.title=Traefik"], new_command.remove_image
+  end
+
+  test "traefik follow logs" do
+    assert_equal \
+    "ssh -t root@1.1.1.1 'docker logs traefik -t -n 10 -f 2>&1'", new_command.follow_logs(host: @config[:servers].first)
+  end
+
+  test "traefik follow logs with grep hello!" do
+    assert_equal \
+    "ssh -t root@1.1.1.1 'docker logs traefik -t -n 10 -f 2>&1 | grep \"hello!\"'", new_command.follow_logs(host: @config[:servers].first, grep: 'hello!')
+  end
+
+  private
+    def new_command
+      Mrsk::Commands::Traefik.new(Mrsk::Configuration.new(@config, version: "123"))
+    end
+end

test/configuration/accessory_test.rb

@@ -1,5 +1,4 @@
 require "test_helper"
-require "mrsk/configuration"
 
 class ConfigurationAccessoryTest < ActiveSupport::TestCase
   setup do
@@ -18,8 +17,15 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
             },
             "secret" => [
               "MYSQL_ROOT_PASSWORD"
-            ]
-          }
+            ],
+          },
+          "files" => [
+            "config/mysql/my.cnf:/etc/mysql/my.cnf",
+            "db/structure.sql:/docker-entrypoint-initdb.d/structure.sql"
+          ],
+          "directories" => [
+            "data:/var/lib/mysql"
+          ]
         },
         "redis" => {
           "image" => "redis:latest",
@@ -59,7 +65,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
   test "missing host" do
     @deploy[:accessories]["mysql"]["host"] = nil
     @config = Mrsk::Configuration.new(@deploy)
-    
+
     assert_raises(ArgumentError) do
       @config.accessory(:mysql).host
     end
@@ -83,7 +89,19 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
   end
 
   test "volume args" do
-    assert_equal [], @config.accessory(:mysql).volume_args
+    assert_equal ["--volume", "$PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf", "--volume", "$PWD/app-mysql/docker-entrypoint-initdb.d/structure.sql:/docker-entrypoint-initdb.d/structure.sql", "--volume", "$PWD/app-mysql/data:/var/lib/mysql"], @config.accessory(:mysql).volume_args
     assert_equal ["--volume", "/var/lib/redis:/data"], @config.accessory(:redis).volume_args
   end
+
+  test "dynamic file expansion" do
+    @deploy[:accessories]["mysql"]["files"] << "test/fixtures/files/structure.sql.erb:/docker-entrypoint-initdb.d/structure.sql"
+    @config = Mrsk::Configuration.new(@deploy)
+
+    assert_match "This was dynamically expanded", @config.accessory(:mysql).files.keys[2].read
+    assert_match "%", @config.accessory(:mysql).files.keys[2].read
+  end
+
+  test "directories" do
+    assert_equal({"$PWD/app-mysql/data"=>"/var/lib/mysql"}, @config.accessory(:mysql).directories)
+  end
 end

test/configuration/role_test.rb

@@ -1,5 +1,4 @@
 require "test_helper"
-require "mrsk/configuration"
 
 class ConfigurationRoleTest < ActiveSupport::TestCase
   setup do
@@ -63,7 +62,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
   end
 
   test "default traefik label on non-web role" do
-    config = Mrsk::Configuration.new(@deploy_with_roles.tap { |c| 
+    config = Mrsk::Configuration.new(@deploy_with_roles.tap { |c|
       c[:servers]["beta"] = { "traefik" => "true", "hosts" => [ "1.1.1.5" ] }
     })
 
@@ -97,7 +96,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
 
     ENV["REDIS_PASSWORD"] = "secret456"
     ENV["DB_PASSWORD"] = "secret123"
-    
+
     assert_equal ["-e", "REDIS_PASSWORD=secret456", "-e", "DB_PASSWORD=secret123", "-e", "REDIS_URL=redis://a/b", "-e", "WEB_CONCURRENCY=4"], @config_with_roles.role(:workers).env_args
   ensure
     ENV["REDIS_PASSWORD"] = nil
@@ -116,7 +115,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     }
 
     ENV["DB_PASSWORD"] = "secret123"
-    
+
     assert_equal ["-e", "DB_PASSWORD=secret123", "-e", "REDIS_URL=redis://a/b", "-e", "WEB_CONCURRENCY=4"], @config_with_roles.role(:workers).env_args
   ensure
     ENV["DB_PASSWORD"] = nil
@@ -133,7 +132,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     }
 
     ENV["REDIS_PASSWORD"] = "secret456"
-    
+
     assert_equal ["-e", "REDIS_PASSWORD=secret456", "-e", "REDIS_URL=redis://a/b", "-e", "WEB_CONCURRENCY=4"], @config_with_roles.role(:workers).env_args
   ensure
     ENV["REDIS_PASSWORD"] = nil

test/configuration_test.rb

@@ -1,10 +1,9 @@
 require "test_helper"
-require "mrsk/configuration"
-
-ENV["RAILS_MASTER_KEY"] = "456"
 
 class ConfigurationTest < ActiveSupport::TestCase
   setup do
+    ENV["RAILS_MASTER_KEY"] = "456"
+
     @deploy = {
       service: "app", image: "dhh/app",
       registry: { "username" => "dhh", "password" => "secret" },
@@ -21,6 +20,10 @@ class ConfigurationTest < ActiveSupport::TestCase
     @config_with_roles = Mrsk::Configuration.new(@deploy_with_roles)
   end
 
+  teardown do
+    ENV["RAILS_MASTER_KEY"] = nil
+  end
+
   test "ensure valid keys" do
     assert_raise(ArgumentError) do
       Mrsk::Configuration.new(@deploy.tap { _1.delete(:service) })
@@ -101,6 +104,14 @@ class ConfigurationTest < ActiveSupport::TestCase
     ENV["PASSWORD"] = nil
   end
 
+  test "env args with only clear" do
+    config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
+      env: { "clear" => { "PORT" => "3000" } }
+    }) })
+
+    assert_equal [ "-e", "PORT=3000" ], config.env_args
+  end
+
   test "env args with only secrets" do
     ENV["PASSWORD"] = "secret123"
     config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
@@ -114,24 +125,32 @@ class ConfigurationTest < ActiveSupport::TestCase
   end
 
   test "env args with missing secret" do
-    config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
-      env: { "secret" => [ "PASSWORD" ] }
-    }) })
-
     assert_raises(KeyError) do
-      assert_equal [ "-e", "PASSWORD=secret123" ], config.env_args
+      config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!({
+        env: { "secret" => [ "PASSWORD" ] }
+      }) })
     end
   end
 
+  test "valid config" do
+    assert @config.valid?
+  end
+
   test "ssh options" do
     assert_equal "root", @config.ssh_options[:user]
 
-    config = Mrsk::Configuration.new(@deploy.tap { |c| c[:ssh_user] = "app" })
+    config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "user" => "app" }) })
     assert_equal "app", @config.ssh_options[:user]
   end
 
-  test "master key" do
-    assert_equal "456", @config.master_key
+  test "ssh options with proxy host" do
+    config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "proxy" => "1.2.3.4" }) })
+    assert_equal "root@1.2.3.4", @config.ssh_options[:proxy].jump_proxies
+  end
+
+  test "ssh options with proxy host and user" do
+    config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "proxy" => "app@1.2.3.4" }) })
+    assert_equal "app@1.2.3.4", @config.ssh_options[:proxy].jump_proxies
   end
 
   test "volume_args" do

test/fixtures/deploy_with_accessories.yml

@@ -17,11 +17,13 @@ accessories:
         MYSQL_ROOT_HOST: '%'
       secret:
         - MYSQL_ROOT_PASSWORD
-    volumes:
-      - /var/lib/mysql:/var/lib/mysql
+    files:
+      - test/fixtures/files/my.cnf:/etc/mysql/my.cnf
+    directories:
+      - data:/var/lib/mysql
   redis:
     image: redis:latest
     host: 1.1.1.4
     port: 6379
-    volumes:
-      - /var/lib/redis:/data
+    directories:
+      - data:/data

test/fixtures/files/my.cnf

@@ -0,0 +1 @@
+# MySQL Config

test/fixtures/files/structure.sql.erb

@@ -0,0 +1,2 @@
+<%= "This was dynamically expanded" %>
+<%= ENV["MYSQL_ROOT_HOST"] %>

test/test_helper.rb

@@ -2,15 +2,14 @@ require "bundler/setup"
 require "active_support/test_case"
 require "active_support/testing/autorun"
 require "debug"
+require "mocha/minitest" # using #stubs that can alter returns
+require "minitest/autorun" # using #stub that take args
 require "sshkit"
+require "mrsk"
 
 ActiveSupport::LogSubscriber.logger = ActiveSupport::Logger.new(STDOUT) if ENV["VERBOSE"]
 
 SSHKit.config.backend = SSHKit::Backend::Printer
 
 class ActiveSupport::TestCase
-  private
-    def stdouted
-      capture(:stdout) { yield }.strip
-    end
 end