Merge pull request #23 from rails/accessories

Accessories
This commit is contained in:
David Heinemeier Hansson
2023-01-22 22:02:04 +01:00
committed by GitHub
16 changed files with 513 additions and 25 deletions

View File

@@ -244,6 +244,33 @@ ARG RUBY_VERSION
FROM ruby:$RUBY_VERSION-slim as base FROM ruby:$RUBY_VERSION-slim as base
``` ```
### Using accessories for database, cache, search services
You can manage your accessory services via MRSK as well. The services will build off public images, and will not be automatically updated when you deploy:
```yaml
accessories:
mysql:
image: mysql:5.7
host: 1.1.1.3
port: 3306
env:
clear:
MYSQL_ROOT_HOST: '%'
secret:
- MYSQL_ROOT_PASSWORD
volumes:
- /var/lib/mysql:/var/lib/mysql
redis:
image: redis:latest
host: 1.1.1.4
port: "36379:6379"
volumes:
- /var/lib/redis:/data
```
Now run `mrsk accessory start mysql` to start the MySQL server on 1.1.1.3. See `mrsk accessory` for all the commands possible.
## Commands ## Commands
### Running remote execution and runners ### Running remote execution and runners

85
lib/mrsk/cli/accessory.rb Normal file
View File

@@ -0,0 +1,85 @@
require "mrsk/cli/base"
class Mrsk::Cli::Accessory < Mrsk::Cli::Base
desc "boot [NAME]", "Boot accessory service on host"
def boot(name)
accessory = MRSK.accessory(name)
on(accessory.host) { execute *accessory.run }
end
desc "reboot [NAME]", "Reboot accessory on host (stop container, remove container, start new container)"
def reboot(name)
invoke :stop, [ name ]
invoke :remove_container, [ name ]
invoke :boot, [ name ]
end
desc "start [NAME]", "Start existing accessory on host"
def start(name)
accessory = MRSK.accessory(name)
on(accessory.host) { execute *accessory.start }
end
desc "stop [NAME]", "Stop accessory on host"
def stop(name)
accessory = MRSK.accessory(name)
on(accessory.host) { execute *accessory.stop }
end
desc "restart [NAME]", "Restart accessory on host"
def restart(name)
invoke :stop, [ name ]
invoke :start, [ name ]
end
desc "details [NAME]", "Display details about accessory on host"
def details(name)
accessory = MRSK.accessory(name)
on(accessory.host) { puts capture_with_info(*accessory.info) }
end
desc "logs [NAME]", "Show log lines from accessory on host"
option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server"
option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
def logs(name)
accessory = MRSK.accessory(name)
grep = options[:grep]
if options[:follow]
run_locally do
info "Following logs on #{accessory.host}..."
info accessory.follow_logs(grep: grep)
exec accessory.follow_logs(grep: grep)
end
else
since = options[:since]
lines = options[:lines]
on(accessory.host) do
puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep))
end
end
end
desc "remove [NAME]", "Remove accessory container and image from host"
def remove(name)
invoke :stop, [ name ]
invoke :remove_container, [ name ]
invoke :remove_image, [ name ]
end
desc "remove_container [NAME]", "Remove accessory container from host"
def remove_container(name)
accessory = MRSK.accessory(name)
on(accessory.host) { execute *accessory.remove_container }
end
desc "remove_container [NAME]", "Remove accessory image from servers"
def remove_image(name)
accessory = MRSK.accessory(name)
on(accessory.host) { execute *accessory.remove_image }
end
end

View File

@@ -1,5 +1,6 @@
require "mrsk/cli/base" require "mrsk/cli/base"
require "mrsk/cli/accessory"
require "mrsk/cli/app" require "mrsk/cli/app"
require "mrsk/cli/build" require "mrsk/cli/build"
require "mrsk/cli/prune" require "mrsk/cli/prune"
@@ -86,6 +87,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
puts Mrsk::VERSION puts Mrsk::VERSION
end end
desc "accessory", "Manage the accessories"
subcommand "accessory", Mrsk::Cli::Accessory
desc "app", "Manage the application" desc "app", "Manage the application"
subcommand "app", Mrsk::Cli::App subcommand "app", Mrsk::Cli::App

View File

@@ -1,6 +1,7 @@
require "active_support/core_ext/enumerable" require "active_support/core_ext/enumerable"
require "mrsk/configuration" require "mrsk/configuration"
require "mrsk/commands/accessory"
require "mrsk/commands/app" require "mrsk/commands/app"
require "mrsk/commands/builder" require "mrsk/commands/builder"
require "mrsk/commands/prune" require "mrsk/commands/prune"
@@ -43,6 +44,10 @@ class Mrsk::Commander
specific_hosts || config.traefik_hosts specific_hosts || config.traefik_hosts
end end
def accessory_hosts
specific_hosts || config.accessories.collect(&:host)
end
def app def app
@app ||= Mrsk::Commands::App.new(config) @app ||= Mrsk::Commands::App.new(config)
@@ -64,6 +69,10 @@ class Mrsk::Commander
@prune ||= Mrsk::Commands::Prune.new(config) @prune ||= Mrsk::Commands::Prune.new(config)
end end
def accessory(name)
(@accessories ||= {})[name] ||= Mrsk::Commands::Accessory.new(config, name: name)
end
def verbosity(level) def verbosity(level)
old_level = SSHKit.config.output_verbosity old_level = SSHKit.config.output_verbosity

View File

@@ -0,0 +1,61 @@
require "mrsk/commands/base"
class Mrsk::Commands::Accessory < Mrsk::Commands::Base
attr_reader :accessory_config
delegate :service_name, :image, :host, :port, :env_args, :volume_args, :label_args, to: :accessory_config
def initialize(config, name:)
super(config)
@accessory_config = config.accessory(name)
end
def run
docker :run,
"--name", service_name,
"-d",
"--restart", "unless-stopped",
"-p", port,
*env_args,
*volume_args,
*label_args,
image
end
def start
docker :container, :start, service_name
end
def stop
docker :container, :stop, service_name
end
def info
docker :ps, *service_filter
end
def logs(since: nil, lines: nil, grep: nil)
pipe \
docker(:logs, service_name, (" --since #{since}" if since), (" -n #{lines}" if lines), "-t", "2>&1"),
("grep '#{grep}'" if grep)
end
def follow_logs(grep: nil)
run_over_ssh pipe(
docker(:logs, service_name, "-t", "-n", "10", "-f", "2>&1"),
("grep '#{grep}'" if grep)
).join(" "), host: host
end
def remove_container
docker :container, :prune, "-f", *service_filter
end
def remove_image
docker :image, :prune, "-a", "-f", *service_filter
end
private
def service_filter
[ "--filter", "label=service=#{service_name}" ]
end
end

View File

@@ -6,8 +6,10 @@ require "erb"
require "mrsk/utils" require "mrsk/utils"
class Mrsk::Configuration class Mrsk::Configuration
delegate :service, :image, :servers, :env, :labels, :registry, :builder, to: :config, allow_nil: true delegate :service, :image, :servers, :env, :labels, :registry, :builder, to: :raw_config, allow_nil: true
delegate :argumentize_env_with_secrets, to: Mrsk::Utils delegate :argumentize, :argumentize_env_with_secrets, to: Mrsk::Utils
attr_accessor :raw_config
class << self class << self
def create_from(base_config_file, destination: nil, version: "missing") def create_from(base_config_file, destination: nil, version: "missing")
@@ -34,8 +36,8 @@ class Mrsk::Configuration
end end
end end
def initialize(config, version: "missing", validate: true) def initialize(raw_config, version: "missing", validate: true)
@config = ActiveSupport::InheritableOptions.new(config) @raw_config = ActiveSupport::InheritableOptions.new(raw_config)
@version = version @version = version
ensure_required_keys_present if validate ensure_required_keys_present if validate
end end
@@ -49,6 +51,15 @@ class Mrsk::Configuration
roles.detect { |r| r.name == name.to_s } roles.detect { |r| r.name == name.to_s }
end end
def accessories
@accessories ||= raw_config.accessories.keys.collect { |name| Mrsk::Configuration::Assessory.new(name, config: self) }
end
def accessory(name)
accessories.detect { |a| a.name == name.to_s }
end
def all_hosts def all_hosts
roles.flat_map(&:hosts) roles.flat_map(&:hosts)
end end
@@ -67,7 +78,7 @@ class Mrsk::Configuration
end end
def repository def repository
[ config.registry["server"], image ].compact.join("/") [ raw_config.registry["server"], image ].compact.join("/")
end end
def absolute_image def absolute_image
@@ -80,23 +91,23 @@ class Mrsk::Configuration
def env_args def env_args
if config.env.present? if raw_config.env.present?
argumentize_env_with_secrets(config.env) argumentize_env_with_secrets(raw_config.env)
else else
[] []
end end
end end
def volume_args def volume_args
if config.volumes.present? if raw_config.volumes.present?
config.volumes.map { |volume| "--volume #{volume}" } argumentize "--volume", raw_config.volumes
else else
[] []
end end
end end
def ssh_user def ssh_user
config.ssh_user || "root" raw_config.ssh_user || "root"
end end
def ssh_options def ssh_options
@@ -117,33 +128,32 @@ class Mrsk::Configuration
absolute_image: absolute_image, absolute_image: absolute_image,
service_with_version: service_with_version, service_with_version: service_with_version,
env_args: env_args, env_args: env_args,
volume_args: volume_args,
ssh_options: ssh_options, ssh_options: ssh_options,
builder: config.builder, builder: raw_config.builder
volume_args: volume_args
}.compact }.compact
end end
private private
attr_accessor :config
def ensure_required_keys_present def ensure_required_keys_present
%i[ service image registry servers ].each do |key| %i[ service image registry servers ].each do |key|
raise ArgumentError, "Missing required configuration for #{key}" unless config[key].present? raise ArgumentError, "Missing required configuration for #{key}" unless raw_config[key].present?
end end
if config.registry["username"].blank? if raw_config.registry["username"].blank?
raise ArgumentError, "You must specify a username for the registry in config/deploy.yml" raise ArgumentError, "You must specify a username for the registry in config/deploy.yml"
end end
if config.registry["password"].blank? if raw_config.registry["password"].blank?
raise ArgumentError, "You must specify a password for the registry in config/deploy.yml (or set the ENV variable if that's used)" raise ArgumentError, "You must specify a password for the registry in config/deploy.yml (or set the ENV variable if that's used)"
end end
end end
def role_names def role_names
config.servers.is_a?(Array) ? [ "web" ] : config.servers.keys.sort raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort
end end
end end
require "mrsk/configuration/role" require "mrsk/configuration/role"
require "mrsk/configuration/accessory"

View File

@@ -0,0 +1,60 @@
class Mrsk::Configuration::Assessory
delegate :argumentize, :argumentize_env_with_secrets, to: Mrsk::Utils
attr_accessor :name, :specifics
def initialize(name, config:)
@name, @config, @specifics = name.inquiry, config, config.raw_config["accessories"][name]
end
def service_name
"#{config.service}-#{name}"
end
def image
specifics["image"]
end
def host
specifics["host"] || raise(ArgumentError, "Missing host for accessory")
end
def port
if specifics["port"].to_s.include?(":")
specifics["port"]
else
"#{specifics["port"]}:#{specifics["port"]}"
end
end
def labels
default_labels.merge(specifics["labels"] || {})
end
def label_args
argumentize "--label", labels
end
def env
specifics["env"] || {}
end
def env_args
argumentize_env_with_secrets env
end
def volumes
specifics["volumes"] || []
end
def volume_args
argumentize "--volume", volumes
end
private
attr_accessor :config
def default_labels
{ "service" => service_name }
end
end

View File

@@ -1,9 +1,15 @@
module Mrsk::Utils module Mrsk::Utils
extend self extend self
# Return a list of shell arguments using the same named argument against the passed attributes. # Return a list of shell arguments using the same named argument against the passed attributes (hash or array).
def argumentize(argument, attributes, redacted: false) def argumentize(argument, attributes, redacted: false)
Array(attributes).flat_map { |k, v| [ argument, redacted ? redact("#{k}=#{v}") : "#{k}=#{v}" ] } Array(attributes).flat_map do |k, v|
if v.present?
[ argument, redacted ? redact("#{k}=#{v}") : "#{k}=#{v}" ]
else
[ argument, k ]
end
end
end end
# Return a list of shell arguments using the same named argument against the passed attributes, # Return a list of shell arguments using the same named argument against the passed attributes,

View File

@@ -0,0 +1,17 @@
require "test_helper"
require "active_support/testing/stream"
require "mrsk/cli"
class CliAccessoryTest < ActiveSupport::TestCase
include ActiveSupport::Testing::Stream
test "boot" do
ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
command = stdouted { Mrsk::Cli::Accessory.start(["boot", "mysql", "-c", "test/fixtures/deploy_with_accessories.yml"]) }
assert_match "Running docker run --name app-mysql -d --restart unless-stopped -p 3306:3306 -e [REDACTED] -e MYSQL_ROOT_HOST=% --volume /var/lib/mysql:/var/lib/mysql --label service=app-mysql mysql:5.7 on 1.1.1.3", command
ensure
ENV["MYSQL_ROOT_PASSWORD"] = nil
end
end

View File

@@ -2,14 +2,14 @@ require "test_helper"
require "active_support/testing/stream" require "active_support/testing/stream"
require "mrsk/cli" require "mrsk/cli"
class CommandsAppTest < ActiveSupport::TestCase class CliMainTest < ActiveSupport::TestCase
include ActiveSupport::Testing::Stream include ActiveSupport::Testing::Stream
setup do setup do
end end
test "version" do test "version" do
version = capture(:stdout) { Mrsk::Cli::Main.new.version }.strip version = stdouted { Mrsk::Cli::Main.new.version }
assert_equal Mrsk::VERSION, version assert_equal Mrsk::VERSION, version
end end
end end

View File

@@ -0,0 +1,86 @@
require "test_helper"
require "mrsk/configuration"
require "mrsk/commands/accessory"
class CommandsAccessoryTest < ActiveSupport::TestCase
setup do
@config = {
service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
servers: [ "1.1.1.1" ],
accessories: {
"mysql" => {
"image" => "mysql:8.0",
"host" => "1.1.1.5",
"port" => "3306",
"env" => {
"clear" => {
"MYSQL_ROOT_HOST" => "%"
},
"secret" => [
"MYSQL_ROOT_PASSWORD"
]
}
},
"redis" => {
"image" => "redis:latest",
"host" => "1.1.1.6",
"port" => "6379:6379",
"labels" => {
"cache" => true
},
"env" => {
"SOMETHING" => "else"
},
"volumes" => [
"/var/lib/redis:/data"
]
}
}
}
@config = Mrsk::Configuration.new(@config)
@mysql = Mrsk::Commands::Accessory.new(@config, name: :mysql)
@redis = Mrsk::Commands::Accessory.new(@config, name: :redis)
end
test "run" do
ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
assert_equal \
[:docker, :run, "--name", "app-mysql", "-d", "--restart", "unless-stopped", "-p", "3306:3306", "-e", "MYSQL_ROOT_PASSWORD=secret123", "-e", "MYSQL_ROOT_HOST=%", "--label", "service=app-mysql", "mysql:8.0"], @mysql.run
assert_equal \
[:docker, :run, "--name", "app-redis", "-d", "--restart", "unless-stopped", "-p", "6379:6379", "-e", "SOMETHING=else", "--volume", "/var/lib/redis:/data", "--label", "service=app-redis", "--label", "cache=true", "redis:latest"], @redis.run
ensure
ENV["MYSQL_ROOT_PASSWORD"] = nil
end
test "start" do
assert_equal [:docker, :container, :start, "app-mysql"], @mysql.start
end
test "stop" do
assert_equal [:docker, :container, :stop, "app-mysql"], @mysql.stop
end
test "info" do
assert_equal [:docker, :ps, "--filter", "label=service=app-mysql"], @mysql.info
end
test "logs" do
assert_equal [:docker, :logs, "app-mysql", "-t", "2>&1"], @mysql.logs
assert_equal [:docker, :logs, "app-mysql", " --since 5m", " -n 100", "-t", "2>&1", "|", "grep 'thing'"], @mysql.logs(since: "5m", lines: 100, grep: "thing")
end
test "follow logs" do
assert_equal "ssh -t root@1.1.1.5 'docker logs app-mysql -t -n 10 -f 2>&1'", @mysql.follow_logs
end
test "remove container" do
assert_equal [:docker, :container, :prune, "-f", "--filter", "label=service=app-mysql"], @mysql.remove_container
end
test "remove image" do
assert_equal [:docker, :image, :prune, "-a", "-f", "--filter", "label=service=app-mysql"], @mysql.remove_image
end
end

View File

@@ -19,7 +19,7 @@ class CommandsAppTest < ActiveSupport::TestCase
@config[:volumes] = ["/local/path:/container/path" ] @config[:volumes] = ["/local/path:/container/path" ]
assert_equal \ assert_equal \
[:docker, :run, "-d", "--restart unless-stopped", "--name", "app-missing", "-e", "RAILS_MASTER_KEY=456", "--volume /local/path:/container/path", "--label", "service=app", "--label", "role=web", "--label", "traefik.http.routers.app.rule='PathPrefix(`/`)'", "--label", "traefik.http.services.app.loadbalancer.healthcheck.path=/up", "--label", "traefik.http.services.app.loadbalancer.healthcheck.interval=1s", "--label", "traefik.http.middlewares.app.retry.attempts=3", "--label", "traefik.http.middlewares.app.retry.initialinterval=500ms", "dhh/app:missing"], @app.run [:docker, :run, "-d", "--restart unless-stopped", "--name", "app-missing", "-e", "RAILS_MASTER_KEY=456", "--volume", "/local/path:/container/path", "--label", "service=app", "--label", "role=web", "--label", "traefik.http.routers.app.rule='PathPrefix(`/`)'", "--label", "traefik.http.services.app.loadbalancer.healthcheck.path=/up", "--label", "traefik.http.services.app.loadbalancer.healthcheck.interval=1s", "--label", "traefik.http.middlewares.app.retry.attempts=3", "--label", "traefik.http.middlewares.app.retry.initialinterval=500ms", "dhh/app:missing"], @app.run
end end
test "run with" do test "run with" do

View File

@@ -0,0 +1,89 @@
require "test_helper"
require "mrsk/configuration"
class ConfigurationAccessoryTest < ActiveSupport::TestCase
setup do
@deploy = {
service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
servers: [ "1.1.1.1", "1.1.1.2" ],
env: { "REDIS_URL" => "redis://x/y" },
accessories: {
"mysql" => {
"image" => "mysql:8.0",
"host" => "1.1.1.5",
"port" => "3306",
"env" => {
"clear" => {
"MYSQL_ROOT_HOST" => "%"
},
"secret" => [
"MYSQL_ROOT_PASSWORD"
]
}
},
"redis" => {
"image" => "redis:latest",
"host" => "1.1.1.6",
"port" => "6379:6379",
"labels" => {
"cache" => true
},
"env" => {
"SOMETHING" => "else"
},
"volumes" => [
"/var/lib/redis:/data"
]
}
}
}
@config = Mrsk::Configuration.new(@deploy)
end
test "service name" do
assert_equal "app-mysql", @config.accessory(:mysql).service_name
assert_equal "app-redis", @config.accessory(:redis).service_name
end
test "port" do
assert_equal "3306:3306", @config.accessory(:mysql).port
assert_equal "6379:6379", @config.accessory(:redis).port
end
test "host" do
assert_equal "1.1.1.5", @config.accessory(:mysql).host
assert_equal "1.1.1.6", @config.accessory(:redis).host
end
test "missing host" do
@deploy[:accessories]["mysql"]["host"] = nil
@config = Mrsk::Configuration.new(@deploy)
assert_raises(ArgumentError) do
@config.accessory(:mysql).host
end
end
test "label args" do
assert_equal ["--label", "service=app-mysql"], @config.accessory(:mysql).label_args
assert_equal ["--label", "service=app-redis", "--label", "cache=true"], @config.accessory(:redis).label_args
end
test "env args with secret" do
ENV["MYSQL_ROOT_PASSWORD"] = "secret123"
assert_equal ["-e", "MYSQL_ROOT_PASSWORD=secret123", "-e", "MYSQL_ROOT_HOST=%"], @config.accessory(:mysql).env_args
assert @config.accessory(:mysql).env_args[1].is_a?(SSHKit::Redaction)
ensure
ENV["MYSQL_ROOT_PASSWORD"] = nil
end
test "env args without secret" do
assert_equal ["-e", "SOMETHING=else"], @config.accessory(:redis).env_args
end
test "volume args" do
assert_equal [], @config.accessory(:mysql).volume_args
assert_equal ["--volume", "/var/lib/redis:/data"], @config.accessory(:redis).volume_args
end
end

View File

@@ -135,7 +135,7 @@ class ConfigurationTest < ActiveSupport::TestCase
end end
test "volume_args" do test "volume_args" do
assert_equal ["--volume /local/path:/container/path"], @config.volume_args assert_equal ["--volume", "/local/path:/container/path"], @config.volume_args
end end
test "erb evaluation of yml config" do test "erb evaluation of yml config" do
@@ -162,6 +162,6 @@ class ConfigurationTest < ActiveSupport::TestCase
end end
test "to_h" do test "to_h" do
assert_equal({ :roles=>["web"], :hosts=>["1.1.1.1", "1.1.1.2"], :primary_host=>"1.1.1.1", :version=>"missing", :repository=>"dhh/app", :absolute_image=>"dhh/app:missing", :service_with_version=>"app-missing", :env_args=>["-e", "REDIS_URL=redis://x/y"], :ssh_options=>{:user=>"root", :auth_methods=>["publickey"]}, :volume_args=>["--volume /local/path:/container/path"] }, @config.to_h) assert_equal({ :roles=>["web"], :hosts=>["1.1.1.1", "1.1.1.2"], :primary_host=>"1.1.1.1", :version=>"missing", :repository=>"dhh/app", :absolute_image=>"dhh/app:missing", :service_with_version=>"app-missing", :env_args=>["-e", "REDIS_URL=redis://x/y"], :ssh_options=>{:user=>"root", :auth_methods=>["publickey"]}, :volume_args=>["--volume", "/local/path:/container/path"] }, @config.to_h)
end end
end end

View File

@@ -0,0 +1,27 @@
service: app
image: dhh/app
servers:
- 1.1.1.1
- 1.1.1.2
registry:
username: user
password: pw
accessories:
mysql:
image: mysql:5.7
host: 1.1.1.3
port: 3306
env:
clear:
MYSQL_ROOT_HOST: '%'
secret:
- MYSQL_ROOT_PASSWORD
volumes:
- /var/lib/mysql:/var/lib/mysql
redis:
image: redis:latest
host: 1.1.1.4
port: 6379
volumes:
- /var/lib/redis:/data

View File

@@ -2,8 +2,15 @@ require "bundler/setup"
require "active_support/test_case" require "active_support/test_case"
require "active_support/testing/autorun" require "active_support/testing/autorun"
require "debug" require "debug"
require "sshkit"
ActiveSupport::LogSubscriber.logger = ActiveSupport::Logger.new(STDOUT) if ENV["VERBOSE"] ActiveSupport::LogSubscriber.logger = ActiveSupport::Logger.new(STDOUT) if ENV["VERBOSE"]
SSHKit.config.backend = SSHKit::Backend::Printer
class ActiveSupport::TestCase class ActiveSupport::TestCase
private
def stdouted
capture(:stdout) { yield }.strip
end
end end