Limit SSH start concurrency

Starting many (90+) SSH connections has caused us some issues such as
failed DNS lookups and hitting process file descriptor limits.

To mitigate this, patch SSHKit::Backend::Netssh to limit concurrency of
connection starts. We'll default to 30 at a time which seems to work
without issue, but can be configured via:

```
sshkit:
  max_concurrent_starts: 10
```
This commit is contained in:
Donal McBreen
2023-06-22 15:12:43 +01:00
parent edcfc77d95
commit b25cfa178b
6 changed files with 63 additions and 1 deletions

View File

@@ -54,3 +54,51 @@ class SSHKit::Backend::Abstract
end
prepend CommandEnvMerge
end
class SSHKit::Backend::Netssh::Configuration
DEFAULT_MAX_CONCURRENT_STARTS = 30
attr_writer :max_concurrent_starts
def max_concurrent_starts
@max_concurrent_starts ||= DEFAULT_MAX_CONCURRENT_STARTS
end
end
class SSHKit::Backend::Netssh
module LimitConcurrentStartsClass
attr_reader :start_semaphore
def configure(&block)
super &block
# Create this here to avoid lazy creation by multiple threads
@start_semaphore = Concurrent::Semaphore.new(config.max_concurrent_starts)
end
end
class << self
prepend LimitConcurrentStartsClass
end
module LimitConcurrentStartsInstance
private
def with_ssh(&block)
host.ssh_options = self.class.config.ssh_options.merge(host.ssh_options || {})
self.class.pool.with(
method(:start_with_concurrency_limit),
String(host.hostname),
host.username,
host.netssh_options,
&block
)
end
def start_with_concurrency_limit(*args)
self.class.start_semaphore.acquire do
Net::SSH.start(*args)
end
end
end
prepend LimitConcurrentStartsInstance
end