Operate accessories

When you want mysql, redis, and the like under MRSK management

lib/mrsk/cli/accessory.rb

@@ -0,0 +1,88 @@
+require "mrsk/cli/base"
+
+class Mrsk::Cli::Accessory < Mrsk::Cli::Base
+  desc "boot [NAME]", "Boot accessory service on host"
+  def boot(name)
+    accessory = MRSK.accessory(name)
+    on(accessory.host) { execute *accessory.run }
+  end
+
+  desc "reboot [NAME]", "Reboot accessory on host (stop container, remove container, start new container)"
+  def reboot(name)
+    invoke :stop, [ name ]
+    invoke :remove_container, [ name ]
+    invoke :boot, [ name ]
+  end
+
+  desc "start [NAME]", "Start existing accessory on host"
+  def start(name)
+    accessory = MRSK.accessory(name)
+    on(accessory.host) { execute *accessory.start }
+  end
+
+  desc "stop [NAME]", "Stop accessory on host"
+  def stop(name)
+    accessory = MRSK.accessory(name)
+    on(accessory.host) { execute *accessory.stop }
+  end
+
+  desc "restart [NAME]", "Restart accessory on host"
+  def restart(name)
+    invoke :stop, [ name ]
+    invoke :start, [ name ]
+  end
+
+  desc "details", "Display details about all accessory containers on hosts"
+  def details
+    MRSK.config.accessories.each do |accessory|
+      on(accessory.host) do |host|
+        puts_by_host host, capture_with_info(*accessory.info), type: "Accessory: #{accessory.name}"
+      end
+    end
+  end
+
+  desc "logs [NAME]", "Show log lines from accessory on host"
+  option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
+  option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server"
+  option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
+  option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
+  def logs(name)
+    accessory = MRSK.accessory(name)
+
+    grep = options[:grep]
+
+    if options[:follow]
+      run_locally do
+        info "Following logs on #{accessory.host}..."
+        info accessory.follow_logs(grep: grep)
+        exec accessory.follow_logs(grep: grep)
+      end
+    else
+      since = options[:since]
+      lines = options[:lines]
+
+      on(accessory.host) do 
+        capture(*MRSK.traefik.logs(since: since, lines: lines, grep: grep))
+      end
+    end
+  end
+
+  desc "remove [NAME]", "Remove accessory container and image from host"
+  def remove(name)
+    invoke :stop, [ name ]
+    invoke :remove_container, [ name ]
+    invoke :remove_image, [ name ]
+  end
+
+  desc "remove_container [NAME]", "Remove accessory container from host"
+  def remove_container(name)
+    accessory = MRSK.accessory(name)
+    on(accessory.host) { execute *accessory.remove_container }
+  end
+
+  desc "remove_container [NAME]", "Remove accessory image from servers"
+  def remove_image(name)
+    accessory = MRSK.accessory(name)
+    on(accessory.host) { execute *accessory.remove_image }
+  end
+end

lib/mrsk/cli/main.rb

@@ -1,5 +1,6 @@
 require "mrsk/cli/base"
 
+require "mrsk/cli/accessory"
 require "mrsk/cli/app"
 require "mrsk/cli/build"
 require "mrsk/cli/prune"
@@ -86,6 +87,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
     puts Mrsk::VERSION
   end
 
+  desc "accessory", "Manage the accessories"
+  subcommand "accessory", Mrsk::Cli::Accessory
+
   desc "app", "Manage the application"
   subcommand "app", Mrsk::Cli::App
 

lib/mrsk/commander.rb

@@ -1,6 +1,7 @@
 require "active_support/core_ext/enumerable"
 
 require "mrsk/configuration"
+require "mrsk/commands/accessory"
 require "mrsk/commands/app"
 require "mrsk/commands/builder"
 require "mrsk/commands/prune"
@@ -43,6 +44,10 @@ class Mrsk::Commander
     specific_hosts || config.traefik_hosts
   end
 
+  def accessory_hosts
+    specific_hosts || config.accessories.collect(&:host)
+  end
+
 
   def app
     @app ||= Mrsk::Commands::App.new(config)
@@ -64,6 +69,10 @@ class Mrsk::Commander
     @prune ||= Mrsk::Commands::Prune.new(config)
   end
 
+  def accessory(name)
+    (@accessories ||= {})[name] ||= Mrsk::Commands::Accessory.new(config, name: name)
+  end
+
 
   def verbosity(level) 
     old_level = SSHKit.config.output_verbosity

lib/mrsk/commands/accessory.rb

@@ -0,0 +1,55 @@
+require "mrsk/commands/base"
+
+class Mrsk::Commands::Accessory < Mrsk::Commands::Base
+  attr_reader :accessory_config
+  delegate :service_name, :image, :host, :port, :volume_args, :label_args, to: :accessory_config
+
+  def initialize(config, name:)
+    super(config)
+    @accessory_config = config.accessory(name)
+  end
+
+  def run
+    docker :run, 
+      "--name", service_name,
+      "-d",
+      "--restart", "unless-stopped",
+      "-p", port,
+      *volume_args,
+      *label_args,
+      image
+  end
+
+  def start
+    docker :container, :start, service_name
+  end
+
+  def stop
+    docker :container, :stop, service_name
+  end
+
+  def info
+    docker :ps, "--filter", "name=#{service_name}"
+  end
+
+  def logs(since: nil, lines: nil, grep: nil)
+    pipe \
+      docker(:logs, service_name, (" --since #{since}" if since), (" -n #{lines}" if lines), "-t", "2>&1"),
+      ("grep '#{grep}'" if grep)
+  end
+
+  def follow_logs(grep: nil)
+    run_over_ssh pipe(
+      docker(:logs, service_name, "-t", "-n", "10", "-f", "2>&1"),
+      ("grep '#{grep}'" if grep)
+    ).join(" "), host: host
+  end
+
+  def remove_container
+    docker :container, :prune, "-f", "--filter", "label=name=#{service_name}"
+  end
+
+  def remove_image
+    docker :image, :prune, "-a", "-f", "--filter", "label=name=#{service_name}"
+  end
+end

lib/mrsk/configuration.rb

@@ -51,6 +51,15 @@ class Mrsk::Configuration
     roles.detect { |r| r.name == name.to_s }
   end
 
+  def accessories
+    @accessories ||= raw_config.accessories.keys.collect { |name| Mrsk::Configuration::Assessory.new(name, config: self) }
+  end
+
+  def accessory(name)
+    accessories.detect { |a| a.name == name.to_s }
+  end
+
+
   def all_hosts
     roles.flat_map(&:hosts)
   end
@@ -138,3 +147,4 @@ class Mrsk::Configuration
 end
 
 require "mrsk/configuration/role"
+require "mrsk/configuration/accessory"

lib/mrsk/configuration/accessory.rb

@@ -0,0 +1,60 @@
+class Mrsk::Configuration::Assessory
+  delegate :argumentize, :argumentize_env_with_secrets, to: Mrsk::Utils
+
+  attr_accessor :name, :specifics
+
+  def initialize(name, config:)
+    @name, @config, @specifics = name.inquiry, config, config.raw_config["accessories"][name]
+  end
+
+  def service_name
+    "#{config.service}-#{name}"
+  end
+
+  def image
+    specifics["image"]
+  end
+
+  def host
+    specifics["host"] || raise(ArgumentError, "Missing host for accessory")
+  end
+
+  def port
+    if specifics["port"].to_s.include?(":")
+      specifics["port"]
+    else
+      "#{specifics["port"]}:#{specifics["port"]}"
+    end
+  end
+
+  def labels
+    default_labels.merge(specifics["labels"] || {})
+  end
+
+  def label_args
+    argumentize "--label", labels
+  end
+
+  def env
+    specifics["env"] || {}
+  end
+
+  def env_args
+    argumentize_env_with_secrets env
+  end
+
+  def volumes
+    specifics["volumes"] || []
+  end
+
+  def volume_args
+    argumentize "--volume", volumes
+  end
+
+  private
+    attr_accessor :config
+
+    def default_labels
+      { "service" => service_name }
+    end
+end

lib/mrsk/utils.rb

@@ -1,9 +1,15 @@
 module Mrsk::Utils
   extend self
 
-  # Return a list of shell arguments using the same named argument against the passed attributes.
+  # Return a list of shell arguments using the same named argument against the passed attributes (hash or array).
   def argumentize(argument, attributes, redacted: false)
-    Array(attributes).flat_map { |k, v| [ argument, redacted ? redact("#{k}=#{v}") : "#{k}=#{v}" ] }
+    Array(attributes).flat_map do |k, v|
+      if v.present?
+        [ argument, redacted ? redact("#{k}=#{v}") : "#{k}=#{v}" ]
+      else
+        [ argument, k ]
+      end
+    end
   end
 
   # Return a list of shell arguments using the same named argument against the passed attributes,

test/cli/accessory_test.rb

@@ -0,0 +1,13 @@
+require "test_helper"
+require "active_support/testing/stream"
+require "mrsk/cli"
+
+class CliAccessoryTest < ActiveSupport::TestCase
+  include ActiveSupport::Testing::Stream
+
+  test "boot" do
+    command = stdouted { Mrsk::Cli::Accessory.start(["boot", "mysql", "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+    
+    assert_match "Running docker run --name app-mysql -d --restart unless-stopped -p 3306:3306 --volume /var/lib/mysql:/var/lib/mysql --label service=app-mysql mysql:5.7 on 1.1.1.3", command
+  end
+end

test/cli/main_test.rb

@@ -9,7 +9,7 @@ class CliMainTest < ActiveSupport::TestCase
   end
 
   test "version" do
-    version = capture(:stdout) { Mrsk::Cli::Main.new.version }.strip
+    version = stdouted { Mrsk::Cli::Main.new.version }
     assert_equal Mrsk::VERSION, version
   end
 end

test/commands/accessory_test.rb

@@ -0,0 +1,82 @@
+require "test_helper"
+require "mrsk/configuration"
+require "mrsk/commands/accessory"
+
+class CommandsAccessoryTest < ActiveSupport::TestCase
+  setup do
+    @config = { 
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+      servers: [ "1.1.1.1" ],
+      accessories: {
+        "mysql" => {
+          "image" => "mysql:8.0",
+          "host" => "1.1.1.5",
+          "port" => "3306",
+          "env" => {
+            "clear" => {
+              "MYSQL_ROOT_HOST" => "%"
+            },
+            "secret" => [
+              "MYSQL_ROOT_PASSWORD"
+            ]
+          }
+        },
+        "redis" => {
+          "image" => "redis:latest",
+          "host" => "1.1.1.6",
+          "port" => "6379:6379",
+          "labels" => {
+            "cache" => true
+          },
+          "env" => {
+            "SOMETHING" => "else"
+          },
+          "volumes" => [
+            "/var/lib/redis:/data"
+          ]
+        }
+      }
+    }
+
+    @config = Mrsk::Configuration.new(@config)
+    @mysql  = Mrsk::Commands::Accessory.new(@config, name: :mysql)
+    @redis  = Mrsk::Commands::Accessory.new(@config, name: :redis)
+  end
+
+  test "run" do
+    assert_equal \
+      [:docker, :run, "--name", "app-mysql", "-d", "--restart", "unless-stopped", "-p", "3306:3306", "--label", "service=app-mysql", "mysql:8.0"], @mysql.run
+
+    assert_equal \
+      [:docker, :run, "--name", "app-redis", "-d", "--restart", "unless-stopped", "-p", "6379:6379", "--volume", "/var/lib/redis:/data", "--label", "service=app-redis", "--label", "cache=true", "redis:latest"], @redis.run
+  end
+
+  test "start" do
+    assert_equal [:docker, :container, :start, "app-mysql"], @mysql.start
+  end
+
+  test "stop" do
+    assert_equal [:docker, :container, :stop, "app-mysql"], @mysql.stop
+  end
+
+  test "info" do
+    assert_equal [:docker, :ps, "--filter", "name=app-mysql"], @mysql.info
+  end
+
+  test "logs" do
+    assert_equal [:docker, :logs, "app-mysql", "-t", "2>&1"], @mysql.logs
+    assert_equal [:docker, :logs, "app-mysql", " --since 5m", " -n 100", "-t", "2>&1", "|", "grep 'thing'"], @mysql.logs(since: "5m", lines: 100, grep: "thing")
+  end
+
+  test "follow logs" do
+    assert_equal "ssh -t root@1.1.1.5 'docker logs app-mysql -t -n 10 -f 2>&1'", @mysql.follow_logs
+  end
+
+  test "remove container" do
+    assert_equal [:docker, :container, :prune, "-f", "--filter", "label=name=app-mysql"], @mysql.remove_container
+  end
+
+  test "remove image" do
+    assert_equal [:docker, :image, :prune, "-a", "-f", "--filter", "label=name=app-mysql"], @mysql.remove_image
+  end
+end

test/configuration/accessory_test.rb

@@ -0,0 +1,89 @@
+require "test_helper"
+require "mrsk/configuration"
+
+class ConfigurationAccessoryTest < ActiveSupport::TestCase
+  setup do
+    @deploy = {
+      service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
+      servers: [ "1.1.1.1", "1.1.1.2" ],
+      env: { "REDIS_URL" => "redis://x/y" },
+      accessories: {
+        "mysql" => {
+          "image" => "mysql:8.0",
+          "host" => "1.1.1.5",
+          "port" => "3306",
+          "env" => {
+            "clear" => {
+              "MYSQL_ROOT_HOST" => "%"
+            },
+            "secret" => [
+              "MYSQL_ROOT_PASSWORD"
+            ]
+          }
+        },
+        "redis" => {
+          "image" => "redis:latest",
+          "host" => "1.1.1.6",
+          "port" => "6379:6379",
+          "labels" => {
+            "cache" => true
+          },
+          "env" => {
+            "SOMETHING" => "else"
+          },
+          "volumes" => [
+            "/var/lib/redis:/data"
+          ]
+        }
+      }
+    }
+
+    @config = Mrsk::Configuration.new(@deploy)
+  end
+
+  test "service name" do
+    assert_equal "app-mysql", @config.accessory(:mysql).service_name
+    assert_equal "app-redis", @config.accessory(:redis).service_name
+  end
+
+  test "port" do
+    assert_equal "3306:3306", @config.accessory(:mysql).port
+    assert_equal "6379:6379", @config.accessory(:redis).port
+  end
+
+  test "host" do
+    assert_equal "1.1.1.5", @config.accessory(:mysql).host
+    assert_equal "1.1.1.6", @config.accessory(:redis).host
+  end
+
+  test "missing host" do
+    @deploy[:accessories]["mysql"]["host"] = nil
+    @config = Mrsk::Configuration.new(@deploy)
+    
+    assert_raises(ArgumentError) do
+      @config.accessory(:mysql).host
+    end
+  end
+
+  test "label args" do
+    assert_equal ["--label", "service=app-mysql"], @config.accessory(:mysql).label_args
+    assert_equal ["--label", "service=app-redis", "--label", "cache=true"], @config.accessory(:redis).label_args
+  end
+
+  test "env args with secret" do
+    ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
+    assert_equal ["-e", "MYSQL_ROOT_PASSWORD=secret123", "-e", "MYSQL_ROOT_HOST=%"], @config.accessory(:mysql).env_args
+    assert @config.accessory(:mysql).env_args[1].is_a?(SSHKit::Redaction)
+  ensure
+    ENV["MYSQL_ROOT_PASSWORD"] = nil
+  end
+
+  test "env args without secret" do
+    assert_equal ["-e", "SOMETHING=else"], @config.accessory(:redis).env_args
+  end
+
+  test "volume args" do
+    assert_equal [], @config.accessory(:mysql).volume_args
+    assert_equal ["--volume", "/var/lib/redis:/data"], @config.accessory(:redis).volume_args
+  end
+end

test/fixtures/deploy_with_accessories.yml

@@ -0,0 +1,27 @@
+service: app
+image: dhh/app
+servers:
+  - 1.1.1.1
+  - 1.1.1.2
+registry:
+  username: user
+  password: pw
+
+accessories:
+  mysql:
+    image: mysql:5.7
+    host: 1.1.1.3
+    port: 3306
+    env:
+      clear:
+        MYSQL_ROOT_HOST: '%'
+      secret:
+        - MYSQL_ROOT_PASSWORD
+    volumes:
+      - /var/lib/mysql:/var/lib/mysql
+  redis:
+    image: redis:8.0
+    host: 1.1.1.4
+    port: 6379
+    volumes:
+      - /var/lib/redis:/data

test/test_helper.rb

@@ -2,8 +2,15 @@ require "bundler/setup"
 require "active_support/test_case"
 require "active_support/testing/autorun"
 require "debug"
+require "sshkit"
 
 ActiveSupport::LogSubscriber.logger = ActiveSupport::Logger.new(STDOUT) if ENV["VERBOSE"]
 
+SSHKit.config.backend = SSHKit::Backend::Printer
+
 class ActiveSupport::TestCase
+  private
+    def stdouted
+      capture(:stdout) { yield }.strip
+    end
 end