Compare commits

...

3 Commits

Author SHA1 Message Date
Donal McBreen
10f3d33cb5 WIP 2024-09-23 15:18:31 +01:00
Donal McBreen
9fd4001a88 Bind to 127.0.0.1 2024-09-23 15:18:31 +01:00
Donal McBreen
6aa707e233 Use local registry for app images
Allow applications to be deployed without needing to set up a repository
in a remote Docker registry.

If the registry server starts with `localhost`, Kamal will start a local
docker registry on that port and push the app image to it.

Then when pulling the image onto the servers, we use net-ssh to forward
the that port from the app server to the deployment server.

This will allow the deployment server to pull the image from the
registry as if it were local, meaning we don't need to set up a cert.
2024-09-23 15:18:28 +01:00
18 changed files with 230 additions and 48 deletions

View File

@@ -9,6 +9,7 @@ PATH
dotenv (~> 3.1)
ed25519 (~> 1.2)
net-ssh (~> 7.0)
net-ssh-gateway
sshkit (>= 1.23.0, < 2.0)
thor (~> 1.3)
zeitwerk (~> 2.5)
@@ -79,6 +80,8 @@ GEM
net-sftp (4.0.0)
net-ssh (>= 5.0.0, < 8.0.0)
net-ssh (7.2.3)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
nokogiri (1.16.7-arm64-darwin)
racc (~> 1.4)
nokogiri (1.16.7-x86_64-darwin)

View File

@@ -14,6 +14,7 @@ Gem::Specification.new do |spec|
spec.add_dependency "activesupport", ">= 7.0"
spec.add_dependency "sshkit", ">= 1.23.0", "< 2.0"
spec.add_dependency "net-ssh", "~> 7.0"
spec.add_dependency "net-ssh-gateway"
spec.add_dependency "thor", "~> 1.3"
spec.add_dependency "dotenv", "~> 3.1"
spec.add_dependency "zeitwerk", "~> 2.5"

View File

@@ -275,7 +275,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
def prepare(name)
with_accessory(name) do |accessory, hosts|
on(hosts) do
execute *KAMAL.registry.login
execute *KAMAL.registry.login unless KAMAL.config.registry.local?
execute *KAMAL.docker.create_network
rescue SSHKit::Command::Failed => e
raise unless e.message.include?("already exists")

View File

@@ -1,4 +1,5 @@
require "uri"
require "net/ssh"
class Kamal::Cli::Build < Kamal::Cli::Base
class BuildError < StandardError; end
@@ -60,6 +61,8 @@ class Kamal::Cli::Build < Kamal::Cli::Base
desc "pull", "Pull app image from registry onto servers"
def pull
tunnels = Kamal::Cli::Tunnel::RemotePorts.new(KAMAL.hosts, KAMAL.config.registry.local_port).tap(&:open) if KAMAL.config.registry.local?
if (first_hosts = mirror_hosts).any?
#  Pull on a single host per mirror first to seed them
say "Pulling image on #{first_hosts.join(", ")} to seed the #{"mirror".pluralize(first_hosts.count)}...", :magenta
@@ -69,6 +72,8 @@ class Kamal::Cli::Build < Kamal::Cli::Base
else
pull_on_hosts(KAMAL.hosts)
end
ensure
tunnels&.close
end
desc "create", "Create a build setup"
@@ -152,7 +157,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base
end
def pull_on_hosts(hosts)
on(hosts) do
on(hosts) do |host|
execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
execute *KAMAL.builder.pull

View File

@@ -22,7 +22,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
invoke_options = deploy_options
say "Log into image registry...", :magenta
invoke "kamal:cli:registry:login", [], invoke_options.merge(skip_local: options[:skip_push])
invoke "kamal:cli:registry:setup", [], invoke_options.merge(skip_local: options[:skip_push])
if options[:skip_push]
say "Pull app image...", :magenta
@@ -184,7 +184,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
invoke "kamal:cli:app:remove", [], options.without(:confirmed)
invoke "kamal:cli:proxy:remove", [], options.without(:confirmed)
invoke "kamal:cli:accessory:remove", [ "all" ], options
invoke "kamal:cli:registry:logout", [], options.without(:confirmed).merge(skip_local: true)
invoke "kamal:cli:registry:remove", [], options.without(:confirmed).merge(skip_local: true)
end
end
end

View File

@@ -9,7 +9,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
end
on(KAMAL.proxy_hosts) do |host|
execute *KAMAL.registry.login
execute *KAMAL.registry.login unless KAMAL.config.registry.local?
version = capture_with_info(*KAMAL.proxy.version).strip.presence
@@ -33,7 +33,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
run_hook "pre-proxy-reboot", hosts: host_list
on(hosts) do |host|
execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
execute *KAMAL.registry.login
execute *KAMAL.registry.login unless KAMAL.config.registry.local?
"Stopping and removing Traefik on #{host}, if running..."
execute *KAMAL.proxy.cleanup_traefik
@@ -76,7 +76,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
run_hook "pre-proxy-reboot", hosts: host_list
on(hosts) do |host|
execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
execute *KAMAL.registry.login
execute *KAMAL.registry.login unless KAMAL.config.registry.local?
"Stopping and removing Traefik on #{host}, if running..."
execute *KAMAL.proxy.cleanup_traefik

View File

@@ -1,17 +1,25 @@
class Kamal::Cli::Registry < Kamal::Cli::Base
desc "login", "Log in to registry locally and remotely"
desc "login", "Setup local registry or log in to remote registry locally and remotely"
option :skip_local, aliases: "-L", type: :boolean, default: false, desc: "Skip local login"
option :skip_remote, aliases: "-R", type: :boolean, default: false, desc: "Skip remote login"
def login
run_locally { execute *KAMAL.registry.login } unless options[:skip_local]
on(KAMAL.hosts) { execute *KAMAL.registry.login } unless options[:skip_remote]
def setup
if KAMAL.registry.local?
run_locally { execute *KAMAL.registry.setup } unless options[:skip_local]
else
run_locally { execute *KAMAL.registry.login } unless options[:skip_local]
on(KAMAL.hosts) { execute *KAMAL.registry.login } unless options[:skip_remote]
end
end
desc "logout", "Log out of registry locally and remotely"
desc "remove", "Remove local registry or log out of remote registry locally and remotely"
option :skip_local, aliases: "-L", type: :boolean, default: false, desc: "Skip local login"
option :skip_remote, aliases: "-R", type: :boolean, default: false, desc: "Skip remote login"
def logout
run_locally { execute *KAMAL.registry.logout } unless options[:skip_local]
on(KAMAL.hosts) { execute *KAMAL.registry.logout } unless options[:skip_remote]
def remove
if KAMAL.registry.local?
run_locally { execute *KAMAL.registry.remove, raise_on_non_zero_exit: false } unless options[:skip_local]
else
run_locally { execute *KAMAL.registry.logout } unless options[:skip_local]
on(KAMAL.hosts) { execute *KAMAL.registry.logout } unless options[:skip_remote]
end
end
end

View File

@@ -0,0 +1,66 @@
Signal.trap "SIGPROF" do
Thread.list.each do |thread|
puts thread.name
puts thread.backtrace.map { |bt| " #{bt}" }
puts
end
end
require "concurrent/map"
class Kamal::Cli::Tunnel::RemotePorts
attr_reader :hosts, :port
def initialize(hosts, port)
@hosts = hosts
@port = port
@open = false
end
def open
@open = true
@opened = Concurrent::Map.new
@threads = hosts.map do |host|
Thread.new do
Net::SSH.start(host, KAMAL.config.ssh.user) do |ssh|
forwarding = nil
ssh.forward.remote(port, "localhost", port, "localhost") do |actual_remote_port|
forwarding = !!actual_remote_port
:no_exception # will yield the exception on my own thread
end
ssh.loop { forwarding.nil? }
if forwarding
@opened[host] = true
ssh.loop(0.1) { @open }
ssh.forward.cancel_remote(port, "localhost")
ssh.loop(0.1) { ssh.forward.active_remotes.include?([ port, "localhost" ]) }
else
@opened[host] = false
end
end
rescue => e
@opened[host] = false
puts e.message
puts e.backtrace
end
end
loop do
break if @opened.size == hosts.size
sleep 0.1
end
raise "Could not open tunnels on #{opened.reject { |k, v| v }.join(", ")}" unless @opened.values.all?
end
def close
p "Closing"
@open = false
p "Joining"
@threads.each(&:join)
p "Joined"
end
end

View File

@@ -1,5 +1,6 @@
class Kamal::Commands::Registry < Kamal::Commands::Base
delegate :registry, to: :config
delegate :local?, :local_port, to: :registry
def login
docker :login,
@@ -11,4 +12,26 @@ class Kamal::Commands::Registry < Kamal::Commands::Base
def logout
docker :logout, registry.server
end
def setup
combine \
docker(:start, "kamal-docker-registry"),
docker(:run, "--detach", "-p", "127.0.0.1:#{local_port}:5000", "--name", "kamal-docker-registry", "registry:2"),
by: "||"
end
def remove
combine \
docker(:stop, "kamal-docker-registry"),
docker(:rm, "kamal-docker-registry"),
by: "&&"
end
def logout
docker :logout, registry.server
end
def tunnel(host)
run_over_ssh "-R", "#{local_port}:localhost:#{local_port}", host: host
end
end

View File

@@ -21,6 +21,14 @@ class Kamal::Configuration::Registry
lookup("password")
end
def local?
server&.match?("^localhost[:$]")
end
def local_port
local? ? (server.split(":").last.to_i || 80) : nil
end
private
def lookup(key)
if registry_config[key].is_a?(Array)

View File

@@ -15,10 +15,12 @@ class Kamal::Configuration::Validator::Registry < Kamal::Configuration::Validato
with_context(key) do
value = config[key]
error "is required" unless value.present?
unless config["server"]&.match?("^localhost[:$]")
error "is required" unless value.present?
unless value.is_a?(String) || (value.is_a?(Array) && value.size == 1 && value.first.is_a?(String))
error "should be a string or an array with one string (for secret lookup)"
unless value.is_a?(String) || (value.is_a?(Array) && value.size == 1 && value.first.is_a?(String))
error "should be a string or an array with one string (for secret lookup)"
end
end
end
end

View File

@@ -22,7 +22,7 @@ class CliMainTest < CliTestCase
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
# deploy
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: true))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: true))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -46,7 +46,7 @@ class CliMainTest < CliTestCase
with_test_secrets("secrets" => "DB_PASSWORD=secret") do
invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true }
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: false))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -72,7 +72,7 @@ class CliMainTest < CliTestCase
test "deploy with skip_push" do
invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: true))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: true))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -159,7 +159,7 @@ class CliMainTest < CliTestCase
invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, :skip_local => false }
Kamal::Cli::Main.any_instance.expects(:invoke)
.with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
.with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: false))
.raises(RuntimeError)
assert_not KAMAL.holding_lock?
@@ -172,7 +172,7 @@ class CliMainTest < CliTestCase
test "deploy with skipped hooks" do
invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => true }
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: false))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -187,7 +187,7 @@ class CliMainTest < CliTestCase
test "deploy with missing secrets" do
invoke_options = { "config_file" => "test/fixtures/deploy_with_secrets.yml", "version" => "999", "skip_hooks" => false }
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: false))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -289,6 +289,16 @@ class CliMainTest < CliTestCase
end
end
test "remove" do
options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_hooks" => false, "confirmed" => true }
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:remove", [], options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:remove", [], options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:remove", [ "all" ], options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:remove", [], options.merge(skip_local: true))
run_command("remove", "-y")
end
test "details" do
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:details")
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:details")

View File

@@ -1,50 +1,62 @@
require_relative "cli_test_case"
class CliRegistryTest < CliTestCase
test "login" do
run_command("login").tap do |output|
test "setup" do
run_command("setup").tap do |output|
assert_match /docker login -u \[REDACTED\] -p \[REDACTED\] as .*@localhost/, output
assert_match /docker login -u \[REDACTED\] -p \[REDACTED\] on 1.1.1.\d/, output
end
end
test "login skip local" do
run_command("login", "-L").tap do |output|
test "setup skip local" do
run_command("setup", "-L").tap do |output|
assert_no_match /docker login -u \[REDACTED\] -p \[REDACTED\] as .*@localhost/, output
assert_match /docker login -u \[REDACTED\] -p \[REDACTED\] on 1.1.1.\d/, output
end
end
test "login skip remote" do
run_command("login", "-R").tap do |output|
test "setup skip remote" do
run_command("setup", "-R").tap do |output|
assert_match /docker login -u \[REDACTED\] -p \[REDACTED\] as .*@localhost/, output
assert_no_match /docker login -u \[REDACTED\] -p \[REDACTED\] on 1.1.1.\d/, output
end
end
test "logout" do
run_command("logout").tap do |output|
test "remove" do
run_command("remove").tap do |output|
assert_match /docker logout as .*@localhost/, output
assert_match /docker logout on 1.1.1.\d/, output
end
end
test "logout skip local" do
run_command("logout", "-L").tap do |output|
test "remove skip local" do
run_command("remove", "-L").tap do |output|
assert_no_match /docker logout as .*@localhost/, output
assert_match /docker logout on 1.1.1.\d/, output
end
end
test "logout skip remote" do
run_command("logout", "-R").tap do |output|
test "remove skip remote" do
run_command("remove", "-R").tap do |output|
assert_match /docker logout as .*@localhost/, output
assert_no_match /docker logout on 1.1.1.\d/, output
end
end
test "setup local registry" do
run_command("setup", fixture: :with_local_registry).tap do |output|
assert_match /docker start kamal-docker-registry || docker run --detach -p 127.0.0.1:5000:5000 --name kamal-docker-registry registry:2 as .*@localhost/, output
end
end
test "remove local registry" do
run_command("remove", fixture: :with_local_registry).tap do |output|
assert_match /docker stop kamal-docker-registry && docker rm kamal-docker-registry as .*@localhost/, output
end
end
private
def run_command(*command)
stdouted { Kamal::Cli::Registry.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
def run_command(*command, fixture: :with_accessories)
stdouted { Kamal::Cli::Registry.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) }
end
end

View File

@@ -55,6 +55,15 @@ class CommandsRegistryTest < ActiveSupport::TestCase
registry.logout.join(" ")
end
test "registry setup" do
@config[:registry] = { "server" => "localhost:5000" }
assert_equal "docker start kamal-docker-registry || docker run --detach -p 5000:5000 --name kamal-docker-registry registry:2", registry.setup.join(" ")
end
test "registry remove" do
assert_equal "docker stop kamal-docker-registry && docker rm kamal-docker-registry", registry.remove.join(" ")
end
private
def registry
Kamal::Commands::Registry.new Kamal::Configuration.new(@config)

View File

@@ -0,0 +1,37 @@
service: app
image: dhh/app
servers:
web:
- "1.1.1.1"
- "1.1.1.2"
workers:
- "1.1.1.3"
- "1.1.1.4"
registry:
server: localhost:5000
builder:
arch: amd64
accessories:
mysql:
image: mysql:5.7
host: 1.1.1.3
port: 3306
env:
clear:
MYSQL_ROOT_HOST: '%'
secret:
- MYSQL_ROOT_PASSWORD
files:
- test/fixtures/files/my.cnf:/etc/mysql/my.cnf
directories:
- data:/var/lib/mysql
redis:
image: redis:latest
roles:
- web
port: 6379
directories:
- data:/data
readiness_delay: 0

View File

@@ -27,14 +27,14 @@ class AppTest < IntegrationTest
images = kamal :app, :images, capture: true
assert_match "App Host: vm1", images
assert_match "App Host: vm2", images
assert_match /registry:4443\/app\s+#{latest_app_version}/, images
assert_match /registry:4443\/app\s+latest/, images
assert_match /localhost:5000\/app\s+#{latest_app_version}/, images
assert_match /localhost:5000\/app\s+latest/, images
containers = kamal :app, :containers, capture: true
assert_match "App Host: vm1", containers
assert_match "App Host: vm2", containers
assert_match "registry:4443/app:#{latest_app_version}", containers
assert_match "registry:4443/app:latest", containers
assert_match "localhost:5000/app:#{latest_app_version}", containers
assert_match "localhost:5000/app:latest", containers
exec_output = kamal :app, :exec, :ps, capture: true
assert_match "App Host: vm1", exec_output

View File

@@ -26,9 +26,7 @@ readiness_delay: 0
proxy:
host: 127.0.0.1
registry:
server: registry:4443
username: root
password: root
server: localhost:5000
builder:
driver: docker
arch: <%= Kamal::Utils.docker_arch %>

View File

@@ -29,7 +29,7 @@ class MainTest < IntegrationTest
assert_match /App Host: vm1/, details
assert_match /App Host: vm2/, details
assert_match /basecamp\/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION}/, details
assert_match /registry:4443\/app:#{first_version}/, details
assert_match /localhost:5000\/app:#{first_version}/, details
audit = kamal :audit, capture: true
assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
@@ -63,8 +63,8 @@ class MainTest < IntegrationTest
assert_equal [ "vm1", "vm2" ], config[:hosts]
assert_equal "vm1", config[:primary_host]
assert_equal version, config[:version]
assert_equal "registry:4443/app", config[:repository]
assert_equal "registry:4443/app:#{version}", config[:absolute_image]
assert_equal "localhost:5000/app", config[:repository]
assert_equal "localhost:5000/app:#{version}", config[:absolute_image]
assert_equal "app-#{version}", config[:service_with_version]
assert_equal [], config[:volume_args]
assert_equal({ user: "root", port: 22, keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])