From 94d6a763a8895c3d5250470e92d4223d7fecab0e Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Wed, 26 Jul 2023 12:26:23 +0100 Subject: [PATCH] Extract ssh and sshkit configuration --- lib/mrsk/commander.rb | 6 ++--- lib/mrsk/commands/base.rb | 10 ++++----- lib/mrsk/configuration.rb | 37 +++++-------------------------- lib/mrsk/configuration/ssh.rb | 24 ++++++++++++++++++++ lib/mrsk/configuration/sshkit.rb | 20 +++++++++++++++++ lib/mrsk/sshkit_with_ext.rb | 18 +++++++-------- test/configuration/ssh_test.rb | 32 ++++++++++++++++++++++++++ test/configuration/sshkit_test.rb | 27 ++++++++++++++++++++++ test/configuration_test.rb | 29 +----------------------- 9 files changed, 127 insertions(+), 76 deletions(-) create mode 100644 lib/mrsk/configuration/ssh.rb create mode 100644 lib/mrsk/configuration/sshkit.rb create mode 100644 test/configuration/ssh_test.rb create mode 100644 test/configuration/sshkit_test.rb diff --git a/lib/mrsk/commander.rb b/lib/mrsk/commander.rb index 94c0aff2..4812bc62 100644 --- a/lib/mrsk/commander.rb +++ b/lib/mrsk/commander.rb @@ -143,10 +143,10 @@ class Mrsk::Commander private # Lazy setup of SSHKit def configure_sshkit_with(config) - SSHKit::Backend::Netssh.pool.idle_timeout = config.sshkit_pool_idle_timeout + SSHKit::Backend::Netssh.pool.idle_timeout = config.sshkit.pool_idle_timeout SSHKit::Backend::Netssh.configure do |sshkit| - sshkit.max_concurrent_starts = config.sshkit_max_concurrent_starts if config.sshkit_max_concurrent_starts - sshkit.ssh_options = config.ssh_options + sshkit.max_concurrent_starts = config.sshkit.max_concurrent_starts + sshkit.ssh_options = config.ssh.options end SSHKit.config.command_map[:docker] = "docker" # No need to use /usr/bin/env, just clogs up the logs SSHKit.config.output_verbosity = verbosity diff --git a/lib/mrsk/commands/base.rb b/lib/mrsk/commands/base.rb index bc427ddf..831f5994 100644 --- a/lib/mrsk/commands/base.rb +++ b/lib/mrsk/commands/base.rb @@ -13,12 +13,12 @@ module Mrsk::Commands def run_over_ssh(*command, host:) "ssh".tap do |cmd| - if config.ssh_proxy && config.ssh_proxy.is_a?(Net::SSH::Proxy::Jump) - cmd << " -J #{config.ssh_proxy.jump_proxies}" - elsif config.ssh_proxy && config.ssh_proxy.is_a?(Net::SSH::Proxy::Command) - cmd << " -o ProxyCommand='#{config.ssh_proxy.command_line_template}'" + if config.ssh.proxy && config.ssh.proxy.is_a?(Net::SSH::Proxy::Jump) + cmd << " -J #{config.ssh.proxy.jump_proxies}" + elsif config.ssh.proxy && config.ssh.proxy.is_a?(Net::SSH::Proxy::Command) + cmd << " -o ProxyCommand='#{config.ssh.proxy.command_line_template}'" end - cmd << " -t #{config.ssh_user}@#{host} '#{command.join(" ")}'" + cmd << " -t #{config.ssh.user}@#{host} '#{command.join(" ")}'" end end diff --git a/lib/mrsk/configuration.rb b/lib/mrsk/configuration.rb index d6badbf3..b1bb0370 100644 --- a/lib/mrsk/configuration.rb +++ b/lib/mrsk/configuration.rb @@ -135,38 +135,12 @@ class Mrsk::Configuration end - def ssh_user - if raw_config.ssh.present? - raw_config.ssh["user"] || "root" - else - "root" - end + def ssh + Mrsk::Configuration::Ssh.new(config: self) end - def ssh_proxy - if raw_config.ssh.present? && raw_config.ssh["proxy"] - Net::SSH::Proxy::Jump.new \ - raw_config.ssh["proxy"].include?("@") ? raw_config.ssh["proxy"] : "root@#{raw_config.ssh["proxy"]}" - elsif raw_config.ssh.present? && raw_config.ssh["proxy_command"] - Net::SSH::Proxy::Command.new(raw_config.ssh["proxy_command"]) - end - end - - def ssh_options - { user: ssh_user, proxy: ssh_proxy, auth_methods: [ "publickey" ], keepalive: true, keepalive_interval: 30 }.compact - end - - - def sshkit_max_concurrent_starts - raw_config.sshkit["max_concurrent_starts"] if raw_config.sshkit.present? - end - - def sshkit_pool_idle_timeout - if raw_config.sshkit.present? - raw_config.sshkit["pool_idle_timeout"] || 900 - else - 900 - end + def sshkit + Mrsk::Configuration::Sshkit.new(config: self) end @@ -198,7 +172,8 @@ class Mrsk::Configuration service_with_version: service_with_version, env_args: env_args, volume_args: volume_args, - ssh_options: ssh_options, + ssh_options: ssh.options, + sshkit: sshkit.to_h, builder: builder.to_h, accessories: raw_config.accessories, logging: logging_args, diff --git a/lib/mrsk/configuration/ssh.rb b/lib/mrsk/configuration/ssh.rb new file mode 100644 index 00000000..d328a5fa --- /dev/null +++ b/lib/mrsk/configuration/ssh.rb @@ -0,0 +1,24 @@ +class Mrsk::Configuration::Ssh + def initialize(config:) + @config = config.raw_config.ssh || {} + end + + def user + config.fetch("user", "root") + end + + def proxy + if (proxy = config["proxy"]) + Net::SSH::Proxy::Jump.new(proxy.include?("@") ? proxy : "root@#{proxy}") + elsif (proxy_command = config["proxy_command"]) + Net::SSH::Proxy::Command.new(proxy_command) + end + end + + def options + { user: user, proxy: proxy, auth_methods: [ "publickey" ], keepalive: true, keepalive_interval: 30 }.compact + end + + private + attr_accessor :config +end diff --git a/lib/mrsk/configuration/sshkit.rb b/lib/mrsk/configuration/sshkit.rb new file mode 100644 index 00000000..5371e15d --- /dev/null +++ b/lib/mrsk/configuration/sshkit.rb @@ -0,0 +1,20 @@ +class Mrsk::Configuration::Sshkit + def initialize(config:) + @options = config.raw_config.sshkit || {} + end + + def max_concurrent_starts + options.fetch("max_concurrent_starts", 30) + end + + def pool_idle_timeout + options.fetch("pool_idle_timeout", 900) + end + + def to_h + options + end + + private + attr_accessor :options +end diff --git a/lib/mrsk/sshkit_with_ext.rb b/lib/mrsk/sshkit_with_ext.rb index 58b1f4e3..eb8f359b 100644 --- a/lib/mrsk/sshkit_with_ext.rb +++ b/lib/mrsk/sshkit_with_ext.rb @@ -56,13 +56,7 @@ class SSHKit::Backend::Abstract end class SSHKit::Backend::Netssh::Configuration - DEFAULT_MAX_CONCURRENT_STARTS = 30 - - attr_writer :max_concurrent_starts - - def max_concurrent_starts - @max_concurrent_starts ||= DEFAULT_MAX_CONCURRENT_STARTS - end + attr_accessor :max_concurrent_starts end class SSHKit::Backend::Netssh @@ -72,7 +66,9 @@ class SSHKit::Backend::Netssh def configure(&block) super &block # Create this here to avoid lazy creation by multiple threads - @start_semaphore = Concurrent::Semaphore.new(config.max_concurrent_starts) + if config.max_concurrent_starts + @start_semaphore = Concurrent::Semaphore.new(config.max_concurrent_starts) + end end end @@ -94,7 +90,11 @@ class SSHKit::Backend::Netssh end def start_with_concurrency_limit(*args) - self.class.start_semaphore.acquire do + if self.class.start_semaphore + self.class.start_semaphore.acquire do + Net::SSH.start(*args) + end + else Net::SSH.start(*args) end end diff --git a/test/configuration/ssh_test.rb b/test/configuration/ssh_test.rb new file mode 100644 index 00000000..b958309e --- /dev/null +++ b/test/configuration/ssh_test.rb @@ -0,0 +1,32 @@ +require "test_helper" + +class ConfigurationSshTest < ActiveSupport::TestCase + setup do + @deploy = { + service: "app", image: "dhh/app", + registry: { "username" => "dhh", "password" => "secret" }, + env: { "REDIS_URL" => "redis://x/y" }, + servers: [ "1.1.1.1", "1.1.1.2" ], + volumes: ["/local/path:/container/path"] + } + + @config = Mrsk::Configuration.new(@deploy) + end + + test "ssh options" do + assert_equal "root", @config.ssh.options[:user] + + config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "user" => "app" }) }) + assert_equal "app", @config.ssh.options[:user] + end + + test "ssh options with proxy host" do + config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "proxy" => "1.2.3.4" }) }) + assert_equal "root@1.2.3.4", @config.ssh.options[:proxy].jump_proxies + end + + test "ssh options with proxy host and user" do + config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "proxy" => "app@1.2.3.4" }) }) + assert_equal "app@1.2.3.4", @config.ssh.options[:proxy].jump_proxies + end +end diff --git a/test/configuration/sshkit_test.rb b/test/configuration/sshkit_test.rb new file mode 100644 index 00000000..36c0e05d --- /dev/null +++ b/test/configuration/sshkit_test.rb @@ -0,0 +1,27 @@ +require "test_helper" + +class ConfigurationSshkitTest < ActiveSupport::TestCase + setup do + @deploy = { + service: "app", image: "dhh/app", + registry: { "username" => "dhh", "password" => "secret" }, + env: { "REDIS_URL" => "redis://x/y" }, + servers: [ "1.1.1.1", "1.1.1.2" ], + volumes: ["/local/path:/container/path"] + } + + @config = Mrsk::Configuration.new(@deploy) + end + + test "sshkit max concurrent starts" do + assert_equal 30, @config.sshkit.max_concurrent_starts + @config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(sshkit: { "max_concurrent_starts" => 50 }) }) + assert_equal 50, @config.sshkit.max_concurrent_starts + end + + test "sshkit pool idle timeout" do + assert_equal 900, @config.sshkit.pool_idle_timeout + @config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(sshkit: { "pool_idle_timeout" => 600 }) }) + assert_equal 600, @config.sshkit.pool_idle_timeout + end +end diff --git a/test/configuration_test.rb b/test/configuration_test.rb index 47eee321..bbcd3101 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -207,34 +207,6 @@ class ConfigurationTest < ActiveSupport::TestCase end end - test "ssh options" do - assert_equal "root", @config.ssh_options[:user] - - config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "user" => "app" }) }) - assert_equal "app", @config.ssh_options[:user] - end - - test "ssh options with proxy host" do - config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "proxy" => "1.2.3.4" }) }) - assert_equal "root@1.2.3.4", @config.ssh_options[:proxy].jump_proxies - end - - test "ssh options with proxy host and user" do - config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(ssh: { "proxy" => "app@1.2.3.4" }) }) - assert_equal "app@1.2.3.4", @config.ssh_options[:proxy].jump_proxies - end - - test "sshkit max concurrent starts" do - config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(sshkit: { "max_concurrent_starts" => 50 }) }) - assert_equal 50, config.sshkit_max_concurrent_starts - end - - test "sshkit pool idle timeout" do - assert_equal 900, @config.sshkit_pool_idle_timeout - config = Mrsk::Configuration.new(@deploy.tap { |c| c.merge!(sshkit: { "pool_idle_timeout" => 600 }) }) - assert_equal 600, config.sshkit_pool_idle_timeout - end - test "volume_args" do assert_equal ["--volume", "/local/path:/container/path"], @config.volume_args end @@ -287,6 +259,7 @@ class ConfigurationTest < ActiveSupport::TestCase :service_with_version=>"app-missing", :env_args=>["-e", "REDIS_URL=\"redis://x/y\""], :ssh_options=>{ :user=>"root", :auth_methods=>["publickey"], keepalive: true, keepalive_interval: 30 }, + :sshkit=>{}, :volume_args=>["--volume", "/local/path:/container/path"], :builder=>{}, :logging=>["--log-opt", "max-size=\"10m\""],