spacebar/kamal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use local registry for app images

Browse Source 
Allow applications to be deployed without needing to set up a repository
in a remote Docker registry.

If the registry server starts with `localhost`, Kamal will start a local
docker registry on that port and push the app image to it.

Then when pulling the image onto the servers, we use net-ssh to forward
the that port from the app server to the deployment server.

This will allow the deployment server to pull the image from the
registry as if it were local, meaning we don't need to set up a cert.
This commit is contained in:
Donal McBreen
2024-09-16 20:41:54 +01:00
parent e71bfcbadd
commit 6aa707e233
17 changed files with 192 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Gemfile.lock
View File

@@ -9,6 +9,7 @@ PATH
dotenv (~> 3.1)
ed25519 (~> 1.2)
net-ssh (~> 7.0)
net-ssh-gateway
sshkit (>= 1.23.0, < 2.0)
thor (~> 1.3)
zeitwerk (~> 2.5)
@@ -79,6 +80,8 @@ GEM
net-sftp (4.0.0)
net-ssh (>= 5.0.0, < 8.0.0)
net-ssh (7.2.3)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
nokogiri (1.16.7-arm64-darwin)
racc (~> 1.4)
nokogiri (1.16.7-x86_64-darwin)

1
kamal.gemspec
View File

@@ -14,6 +14,7 @@ Gem::Specification.new do |spec|
spec.add_dependency "activesupport", ">= 7.0"
spec.add_dependency "sshkit", ">= 1.23.0", "< 2.0"
spec.add_dependency "net-ssh", "~> 7.0"
spec.add_dependency "net-ssh-gateway"
spec.add_dependency "thor", "~> 1.3"
spec.add_dependency "dotenv", "~> 3.1"
spec.add_dependency "zeitwerk", "~> 2.5"

2
lib/kamal/cli/accessory.rb
View File

@@ -275,7 +275,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
def prepare(name)
with_accessory(name) do |accessory, hosts|
on(hosts) do
execute *KAMAL.registry.login
execute *KAMAL.registry.login unless KAMAL.config.registry.local?
execute *KAMAL.docker.create_network
rescue SSHKit::Command::Failed => e
raise unless e.message.include?("already exists")

7
lib/kamal/cli/build.rb
View File

@@ -1,4 +1,5 @@
require "uri"
require "net/ssh"
class Kamal::Cli::Build < Kamal::Cli::Base
class BuildError < StandardError; end
@@ -60,6 +61,8 @@ class Kamal::Cli::Build < Kamal::Cli::Base
desc "pull", "Pull app image from registry onto servers"
def pull
tunnels = Kamal::Cli::Tunnel::RemotePorts.new(KAMAL.hosts, KAMAL.config.registry.local_port).tap(&:open) if KAMAL.config.registry.local?
if (first_hosts = mirror_hosts).any?
#  Pull on a single host per mirror first to seed them
say "Pulling image on #{first_hosts.join(", ")} to seed the #{"mirror".pluralize(first_hosts.count)}...", :magenta
@@ -69,6 +72,8 @@ class Kamal::Cli::Build < Kamal::Cli::Base
else
pull_on_hosts(KAMAL.hosts)
end
ensure
tunnels&.close
end
desc "create", "Create a build setup"
@@ -152,7 +157,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base
end
def pull_on_hosts(hosts)
on(hosts) do
on(hosts) do |host|
execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
execute *KAMAL.builder.pull

4
lib/kamal/cli/main.rb
View File

@@ -22,7 +22,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
invoke_options = deploy_options
say "Log into image registry...", :magenta
invoke "kamal:cli:registry:login", [], invoke_options.merge(skip_local: options[:skip_push])
invoke "kamal:cli:registry:setup", [], invoke_options.merge(skip_local: options[:skip_push])
if options[:skip_push]
say "Pull app image...", :magenta
@@ -184,7 +184,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
invoke "kamal:cli:app:remove", [], options.without(:confirmed)
invoke "kamal:cli:proxy:remove", [], options.without(:confirmed)
invoke "kamal:cli:accessory:remove", [ "all" ], options
invoke "kamal:cli:registry:logout", [], options.without(:confirmed).merge(skip_local: true)
invoke "kamal:cli:registry:remove", [], options.without(:confirmed).merge(skip_local: true)
end
end
end

6
lib/kamal/cli/proxy.rb
View File

@@ -9,7 +9,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
end
on(KAMAL.proxy_hosts) do |host|
execute *KAMAL.registry.login
execute *KAMAL.registry.login unless KAMAL.config.registry.local?
version = capture_with_info(*KAMAL.proxy.version).strip.presence
@@ -33,7 +33,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
run_hook "pre-proxy-reboot", hosts: host_list
on(hosts) do |host|
execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
execute *KAMAL.registry.login
execute *KAMAL.registry.login unless KAMAL.config.registry.local?
"Stopping and removing Traefik on #{host}, if running..."
execute *KAMAL.proxy.cleanup_traefik
@@ -76,7 +76,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
run_hook "pre-proxy-reboot", hosts: host_list
on(hosts) do |host|
execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
execute *KAMAL.registry.login
execute *KAMAL.registry.login unless KAMAL.config.registry.local?
"Stopping and removing Traefik on #{host}, if running..."
execute *KAMAL.proxy.cleanup_traefik

16
lib/kamal/cli/registry.rb
View File

@@ -1,17 +1,25 @@
class Kamal::Cli::Registry < Kamal::Cli::Base
desc "login", "Log in to registry locally and remotely"
desc "login", "Setup local registry or log in to remote registry locally and remotely"
option :skip_local, aliases: "-L", type: :boolean, default: false, desc: "Skip local login"
option :skip_remote, aliases: "-R", type: :boolean, default: false, desc: "Skip remote login"
def login
def setup
if KAMAL.registry.local?
run_locally { execute *KAMAL.registry.setup } unless options[:skip_local]
else
run_locally { execute *KAMAL.registry.login } unless options[:skip_local]
on(KAMAL.hosts) { execute *KAMAL.registry.login } unless options[:skip_remote]
end
end
desc "logout", "Log out of registry locally and remotely"
desc "remove", "Remove local registry or log out of remote registry locally and remotely"
option :skip_local, aliases: "-L", type: :boolean, default: false, desc: "Skip local login"
option :skip_remote, aliases: "-R", type: :boolean, default: false, desc: "Skip remote login"
def logout
def remove
if KAMAL.registry.local?
run_locally { execute *KAMAL.registry.remove, raise_on_non_zero_exit: false } unless options[:skip_local]
else
run_locally { execute *KAMAL.registry.logout } unless options[:skip_local]
on(KAMAL.hosts) { execute *KAMAL.registry.logout } unless options[:skip_remote]
end
end
end

66
lib/kamal/cli/tunnel/remote_ports.rb Normal file
View File

@@ -0,0 +1,66 @@
Signal.trap "SIGPROF" do
Thread.list.each do |thread|
puts thread.name
puts thread.backtrace.map { |bt| " #{bt}" }
puts
end
end
require "concurrent/map"
class Kamal::Cli::Tunnel::RemotePorts
attr_reader :hosts, :port
def initialize(hosts, port)
@hosts = hosts
@port = port
@open = false
end
def open
@open = true
@opened = Concurrent::Map.new
@threads = hosts.map do |host|
Thread.new do
Net::SSH.start(host, KAMAL.config.ssh.user) do |ssh|
forwarding = nil
ssh.forward.remote(port, "localhost", port, "localhost") do |actual_remote_port|
forwarding = !!actual_remote_port
:no_exception # will yield the exception on my own thread
end
ssh.loop { forwarding.nil? }
if forwarding
@opened[host] = true
ssh.loop(0.1) { @open }
ssh.forward.cancel_remote(port, "localhost")
ssh.loop(0.1) { ssh.forward.active_remotes.include?([ port, "localhost" ]) }
else
@opened[host] = false
end
end
rescue => e
@opened[host] = false
puts e.message
puts e.backtrace
end
end
loop do
break if @opened.size == hosts.size
sleep 0.1
end
raise "Could not open tunnels on #{opened.reject { |k, v| v }.join(", ")}" unless @opened.values.all?
end
def close
p "Closing"
@open = false
p "Joining"
@threads.each(&:join)
p "Joined"
end
end

23
lib/kamal/commands/registry.rb
View File

@@ -1,5 +1,6 @@
class Kamal::Commands::Registry < Kamal::Commands::Base
delegate :registry, to: :config
delegate :local?, :local_port, to: :registry
def login
docker :login,
@@ -11,4 +12,26 @@ class Kamal::Commands::Registry < Kamal::Commands::Base
def logout
docker :logout, registry.server
end
def setup
combine \
docker(:start, "kamal-docker-registry"),
docker(:run, "--detach", "-p", "#{local_port}:5000", "--name", "kamal-docker-registry", "registry:2"),
by: "||"
end
def remove
combine \
docker(:stop, "kamal-docker-registry"),
docker(:rm, "kamal-docker-registry"),
by: "&&"
end
def logout
docker :logout, registry.server
end
def tunnel(host)
run_over_ssh "-R", "#{local_port}:localhost:#{local_port}", host: host
end
end

8
lib/kamal/configuration/registry.rb
View File

@@ -21,6 +21,14 @@ class Kamal::Configuration::Registry
lookup("password")
end
def local?
server&.match?("^localhost[:$]")
end
def local_port
local? ? (server.split(":").last.to_i || 80) : nil
end
private
def lookup(key)
if registry_config[key].is_a?(Array)

2
lib/kamal/configuration/validator/registry.rb
View File

@@ -15,6 +15,7 @@ class Kamal::Configuration::Validator::Registry < Kamal::Configuration::Validato
with_context(key) do
value = config[key]
unless config["server"]&.match?("^localhost[:$]")
error "is required" unless value.present?
unless value.is_a?(String) || (value.is_a?(Array) && value.size == 1 && value.first.is_a?(String))
@@ -22,4 +23,5 @@ class Kamal::Configuration::Validator::Registry < Kamal::Configuration::Validato
end
end
end
end
end

22
test/cli/main_test.rb
View File

@@ -22,7 +22,7 @@ class CliMainTest < CliTestCase
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
# deploy
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: true))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: true))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -46,7 +46,7 @@ class CliMainTest < CliTestCase
with_test_secrets("secrets" => "DB_PASSWORD=secret") do
invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true }
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: false))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -72,7 +72,7 @@ class CliMainTest < CliTestCase
test "deploy with skip_push" do
invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: true))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: true))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -159,7 +159,7 @@ class CliMainTest < CliTestCase
invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, :skip_local => false }
Kamal::Cli::Main.any_instance.expects(:invoke)
.with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
.with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: false))
.raises(RuntimeError)
assert_not KAMAL.holding_lock?
@@ -172,7 +172,7 @@ class CliMainTest < CliTestCase
test "deploy with skipped hooks" do
invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => true }
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: false))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -187,7 +187,7 @@ class CliMainTest < CliTestCase
test "deploy with missing secrets" do
invoke_options = { "config_file" => "test/fixtures/deploy_with_secrets.yml", "version" => "999", "skip_hooks" => false }
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: false))
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -289,6 +289,16 @@ class CliMainTest < CliTestCase
end
end
test "remove" do
options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_hooks" => false, "confirmed" => true }
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:remove", [], options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:remove", [], options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:remove", [ "all" ], options)
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:remove", [], options.merge(skip_local: true))
run_command("remove", "-y")
end
test "details" do
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:details")
Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:details")

40
test/cli/registry_test.rb
View File

@@ -1,50 +1,62 @@
require_relative "cli_test_case"
class CliRegistryTest < CliTestCase
test "login" do
run_command("login").tap do |output|
test "setup" do
run_command("setup").tap do |output|
assert_match /docker login -u \[REDACTED\] -p \[REDACTED\] as .*@localhost/, output
assert_match /docker login -u \[REDACTED\] -p \[REDACTED\] on 1.1.1.\d/, output
end
end
test "login skip local" do
run_command("login", "-L").tap do |output|
test "setup skip local" do
run_command("setup", "-L").tap do |output|
assert_no_match /docker login -u \[REDACTED\] -p \[REDACTED\] as .*@localhost/, output
assert_match /docker login -u \[REDACTED\] -p \[REDACTED\] on 1.1.1.\d/, output
end
end
test "login skip remote" do
run_command("login", "-R").tap do |output|
test "setup skip remote" do
run_command("setup", "-R").tap do |output|
assert_match /docker login -u \[REDACTED\] -p \[REDACTED\] as .*@localhost/, output
assert_no_match /docker login -u \[REDACTED\] -p \[REDACTED\] on 1.1.1.\d/, output
end
end
test "logout" do
run_command("logout").tap do |output|
test "remove" do
run_command("remove").tap do |output|
assert_match /docker logout as .*@localhost/, output
assert_match /docker logout on 1.1.1.\d/, output
end
end
test "logout skip local" do
run_command("logout", "-L").tap do |output|
test "remove skip local" do
run_command("remove", "-L").tap do |output|
assert_no_match /docker logout as .*@localhost/, output
assert_match /docker logout on 1.1.1.\d/, output
end
end
test "logout skip remote" do
run_command("logout", "-R").tap do |output|
test "remove skip remote" do
run_command("remove", "-R").tap do |output|
assert_match /docker logout as .*@localhost/, output
assert_no_match /docker logout on 1.1.1.\d/, output
end
end
test "setup local registry" do
run_command("setup", fixture: :with_local_registry).tap do |output|
assert_match /docker start kamal-docker-registry || docker run --detach -p 5000:5000 --name kamal-docker-registry registry:2 as .*@localhost/, output
end
end
test "remove local registry" do
run_command("remove", fixture: :with_local_registry).tap do |output|
assert_match /docker stop kamal-docker-registry && docker rm kamal-docker-registry as .*@localhost/, output
end
end
private
def run_command(*command)
stdouted { Kamal::Cli::Registry.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
def run_command(*command, fixture: :with_accessories)
stdouted { Kamal::Cli::Registry.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) }
end
end

8
test/commands/registry_test.rb
View File

@@ -55,6 +55,14 @@ class CommandsRegistryTest < ActiveSupport::TestCase
registry.logout.join(" ")
end
test "registry setup" do
assert_equal "docker start kamal-docker-registry || docker run --detach -p :5000 --name kamal-docker-registry registry:2", registry.setup.join(" ")
end
test "registry remove" do
assert_equal "docker stop kamal-docker-registry && docker rm kamal-docker-registry", registry.remove.join(" ")
end
private
def registry
Kamal::Commands::Registry.new Kamal::Configuration.new(@config)

8
test/integration/app_test.rb
View File

@@ -27,14 +27,14 @@ class AppTest < IntegrationTest
images = kamal :app, :images, capture: true
assert_match "App Host: vm1", images
assert_match "App Host: vm2", images
assert_match /registry:4443\/app\s+#{latest_app_version}/, images
assert_match /registry:4443\/app\s+latest/, images
assert_match /localhost:5000\/app\s+#{latest_app_version}/, images
assert_match /localhost:5000\/app\s+latest/, images
containers = kamal :app, :containers, capture: true
assert_match "App Host: vm1", containers
assert_match "App Host: vm2", containers
assert_match "registry:4443/app:#{latest_app_version}", containers
assert_match "registry:4443/app:latest", containers
assert_match "localhost:5000/app:#{latest_app_version}", containers
assert_match "localhost:5000/app:latest", containers
exec_output = kamal :app, :exec, :ps, capture: true
assert_match "App Host: vm1", exec_output

4
test/integration/docker/deployer/app/config/deploy.yml
View File

@@ -26,9 +26,7 @@ readiness_delay: 0
proxy:
host: 127.0.0.1
registry:
server: registry:4443
username: root
password: root
server: localhost:5000
builder:
driver: docker
arch: <%= Kamal::Utils.docker_arch %>

6
test/integration/main_test.rb
View File

@@ -29,7 +29,7 @@ class MainTest < IntegrationTest
assert_match /App Host: vm1/, details
assert_match /App Host: vm2/, details
assert_match /basecamp\/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION}/, details
assert_match /registry:4443\/app:#{first_version}/, details
assert_match /localhost:5000\/app:#{first_version}/, details
audit = kamal :audit, capture: true
assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
@@ -63,8 +63,8 @@ class MainTest < IntegrationTest
assert_equal [ "vm1", "vm2" ], config[:hosts]
assert_equal "vm1", config[:primary_host]
assert_equal version, config[:version]
assert_equal "registry:4443/app", config[:repository]
assert_equal "registry:4443/app:#{version}", config[:absolute_image]
assert_equal "localhost:5000/app", config[:repository]
assert_equal "localhost:5000/app:#{version}", config[:absolute_image]
assert_equal "app-#{version}", config[:service_with_version]
assert_equal [], config[:volume_args]
assert_equal({ user: "root", port: 22, keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])