Use local registry for app images

Allow applications to be deployed without needing to set up a repository
in a remote Docker registry.

If the registry server starts with `localhost`, Kamal will start a local
docker registry on that port and push the app image to it.

Then when pulling the image onto the servers, we use net-ssh to forward
the that port from the app server to the deployment server.

This will allow the deployment server to pull the image from the
registry as if it were local, meaning we don't need to set up a cert.

Gemfile.lock

@@ -9,6 +9,7 @@ PATH
       dotenv (~> 3.1)
       ed25519 (~> 1.2)
       net-ssh (~> 7.0)
+      net-ssh-gateway
       sshkit (>= 1.23.0, < 2.0)
       thor (~> 1.3)
       zeitwerk (~> 2.5)
@@ -79,6 +80,8 @@ GEM
     net-sftp (4.0.0)
       net-ssh (>= 5.0.0, < 8.0.0)
     net-ssh (7.2.3)
+    net-ssh-gateway (2.0.0)
+      net-ssh (>= 4.0.0)
     nokogiri (1.16.7-arm64-darwin)
       racc (~> 1.4)
     nokogiri (1.16.7-x86_64-darwin)

kamal.gemspec

@@ -14,6 +14,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "activesupport", ">= 7.0"
   spec.add_dependency "sshkit", ">= 1.23.0", "< 2.0"
   spec.add_dependency "net-ssh", "~> 7.0"
+  spec.add_dependency "net-ssh-gateway"
   spec.add_dependency "thor", "~> 1.3"
   spec.add_dependency "dotenv", "~> 3.1"
   spec.add_dependency "zeitwerk", "~> 2.5"

lib/kamal/cli/accessory.rb

@@ -275,7 +275,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     def prepare(name)
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
-          execute *KAMAL.registry.login
+          execute *KAMAL.registry.login unless KAMAL.config.registry.local?
           execute *KAMAL.docker.create_network
         rescue SSHKit::Command::Failed => e
           raise unless e.message.include?("already exists")

lib/kamal/cli/build.rb

@@ -1,4 +1,5 @@
 require "uri"
+require "net/ssh"
 
 class Kamal::Cli::Build < Kamal::Cli::Base
   class BuildError < StandardError; end
@@ -60,6 +61,8 @@ class Kamal::Cli::Build < Kamal::Cli::Base
 
   desc "pull", "Pull app image from registry onto servers"
   def pull
+    tunnels = Kamal::Cli::Tunnel::RemotePorts.new(KAMAL.hosts, KAMAL.config.registry.local_port).tap(&:open) if KAMAL.config.registry.local?
+
     if (first_hosts = mirror_hosts).any?
       #  Pull on a single host per mirror first to seed them
       say "Pulling image on #{first_hosts.join(", ")} to seed the #{"mirror".pluralize(first_hosts.count)}...", :magenta
@@ -69,6 +72,8 @@ class Kamal::Cli::Build < Kamal::Cli::Base
     else
       pull_on_hosts(KAMAL.hosts)
     end
+  ensure
+    tunnels&.close
   end
 
   desc "create", "Create a build setup"
@@ -152,7 +157,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base
     end
 
     def pull_on_hosts(hosts)
-      on(hosts) do
+      on(hosts) do |host|
         execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
         execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
         execute *KAMAL.builder.pull

lib/kamal/cli/main.rb

@@ -22,7 +22,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       invoke_options = deploy_options
 
       say "Log into image registry...", :magenta
-      invoke "kamal:cli:registry:login", [], invoke_options.merge(skip_local: options[:skip_push])
+      invoke "kamal:cli:registry:setup", [], invoke_options.merge(skip_local: options[:skip_push])
 
       if options[:skip_push]
         say "Pull app image...", :magenta
@@ -184,7 +184,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
         invoke "kamal:cli:app:remove", [], options.without(:confirmed)
         invoke "kamal:cli:proxy:remove", [], options.without(:confirmed)
         invoke "kamal:cli:accessory:remove", [ "all" ], options
-        invoke "kamal:cli:registry:logout", [], options.without(:confirmed).merge(skip_local: true)
+        invoke "kamal:cli:registry:remove", [], options.without(:confirmed).merge(skip_local: true)
       end
     end
   end

lib/kamal/cli/proxy.rb

@@ -9,7 +9,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
       end
 
       on(KAMAL.proxy_hosts) do |host|
-        execute *KAMAL.registry.login
+        execute *KAMAL.registry.login unless KAMAL.config.registry.local?
 
         version = capture_with_info(*KAMAL.proxy.version).strip.presence
 
@@ -33,7 +33,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
           run_hook "pre-proxy-reboot", hosts: host_list
           on(hosts) do |host|
             execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
-            execute *KAMAL.registry.login
+            execute *KAMAL.registry.login unless KAMAL.config.registry.local?
 
             "Stopping and removing Traefik on #{host}, if running..."
             execute *KAMAL.proxy.cleanup_traefik
@@ -76,7 +76,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
         run_hook "pre-proxy-reboot", hosts: host_list
         on(hosts) do |host|
           execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
-          execute *KAMAL.registry.login
+          execute *KAMAL.registry.login unless KAMAL.config.registry.local?
 
           "Stopping and removing Traefik on #{host}, if running..."
           execute *KAMAL.proxy.cleanup_traefik

lib/kamal/cli/registry.rb

@@ -1,17 +1,25 @@
 class Kamal::Cli::Registry < Kamal::Cli::Base
-  desc "login", "Log in to registry locally and remotely"
+  desc "login", "Setup local registry or log in to remote registry locally and remotely"
   option :skip_local, aliases: "-L", type: :boolean, default: false, desc: "Skip local login"
   option :skip_remote, aliases: "-R", type: :boolean, default: false, desc: "Skip remote login"
-  def login
+  def setup
-    run_locally    { execute *KAMAL.registry.login } unless options[:skip_local]
+    if KAMAL.registry.local?
-    on(KAMAL.hosts) { execute *KAMAL.registry.login } unless options[:skip_remote]
+      run_locally    { execute *KAMAL.registry.setup } unless options[:skip_local]
+    else
+      run_locally    { execute *KAMAL.registry.login } unless options[:skip_local]
+      on(KAMAL.hosts) { execute *KAMAL.registry.login } unless options[:skip_remote]
+    end
   end
 
-  desc "logout", "Log out of registry locally and remotely"
+  desc "remove", "Remove local registry or log out of remote registry locally and remotely"
   option :skip_local, aliases: "-L", type: :boolean, default: false, desc: "Skip local login"
   option :skip_remote, aliases: "-R", type: :boolean, default: false, desc: "Skip remote login"
-  def logout
+  def remove
-    run_locally    { execute *KAMAL.registry.logout } unless options[:skip_local]
+    if KAMAL.registry.local?
-    on(KAMAL.hosts) { execute *KAMAL.registry.logout } unless options[:skip_remote]
+      run_locally    { execute *KAMAL.registry.remove, raise_on_non_zero_exit: false } unless options[:skip_local]
+    else
+      run_locally    { execute *KAMAL.registry.logout } unless options[:skip_local]
+      on(KAMAL.hosts) { execute *KAMAL.registry.logout } unless options[:skip_remote]
+    end
   end
 end

lib/kamal/cli/tunnel/remote_ports.rb

@@ -0,0 +1,66 @@
+Signal.trap "SIGPROF" do
+  Thread.list.each do |thread|
+    puts thread.name
+    puts thread.backtrace.map { |bt| "  #{bt}" }
+    puts
+  end
+end
+
+require "concurrent/map"
+
+class Kamal::Cli::Tunnel::RemotePorts
+  attr_reader :hosts, :port
+
+  def initialize(hosts, port)
+    @hosts = hosts
+    @port = port
+    @open = false
+  end
+
+  def open
+    @open = true
+    @opened = Concurrent::Map.new
+
+    @threads = hosts.map do |host|
+      Thread.new do
+        Net::SSH.start(host, KAMAL.config.ssh.user) do |ssh|
+          forwarding = nil
+          ssh.forward.remote(port, "localhost", port, "localhost") do |actual_remote_port|
+            forwarding = !!actual_remote_port
+            :no_exception # will yield the exception on my own thread
+          end
+          ssh.loop { forwarding.nil? }
+          if forwarding
+            @opened[host] = true
+            ssh.loop(0.1) { @open }
+
+            ssh.forward.cancel_remote(port, "localhost")
+            ssh.loop(0.1) { ssh.forward.active_remotes.include?([ port, "localhost" ]) }
+          else
+            @opened[host] = false
+          end
+        end
+      rescue => e
+        @opened[host] = false
+
+        puts e.message
+        puts e.backtrace
+      end
+    end
+
+    loop do
+      break if @opened.size == hosts.size
+      sleep 0.1
+    end
+
+    raise "Could not open tunnels on #{opened.reject { |k, v| v }.join(", ")}" unless @opened.values.all?
+  end
+
+  def close
+    p "Closing"
+    @open = false
+    p "Joining"
+    @threads.each(&:join)
+    p "Joined"
+  end
+end

lib/kamal/commands/registry.rb

@@ -1,5 +1,6 @@
 class Kamal::Commands::Registry < Kamal::Commands::Base
   delegate :registry, to: :config
+  delegate :local?, :local_port, to: :registry
 
   def login
     docker :login,
@@ -11,4 +12,26 @@ class Kamal::Commands::Registry < Kamal::Commands::Base
   def logout
     docker :logout, registry.server
   end
+
+  def setup
+    combine \
+      docker(:start, "kamal-docker-registry"),
+      docker(:run, "--detach", "-p", "#{local_port}:5000", "--name", "kamal-docker-registry", "registry:2"),
+      by: "||"
+  end
+
+  def remove
+    combine \
+      docker(:stop, "kamal-docker-registry"),
+      docker(:rm, "kamal-docker-registry"),
+      by: "&&"
+  end
+
+  def logout
+    docker :logout, registry.server
+  end
+
+  def tunnel(host)
+    run_over_ssh "-R", "#{local_port}:localhost:#{local_port}", host: host
+  end
 end

lib/kamal/configuration/registry.rb

@@ -21,6 +21,14 @@ class Kamal::Configuration::Registry
     lookup("password")
   end
 
+  def local?
+    server&.match?("^localhost[:$]")
+  end
+
+  def local_port
+    local? ? (server.split(":").last.to_i || 80) : nil
+  end
+
   private
     def lookup(key)
       if registry_config[key].is_a?(Array)

lib/kamal/configuration/validator/registry.rb

@@ -15,10 +15,12 @@ class Kamal::Configuration::Validator::Registry < Kamal::Configuration::Validato
       with_context(key) do
         value = config[key]
 
-        error "is required" unless value.present?
+        unless config["server"]&.match?("^localhost[:$]")
+          error "is required" unless value.present?
 
-        unless value.is_a?(String) || (value.is_a?(Array) && value.size == 1 && value.first.is_a?(String))
+          unless value.is_a?(String) || (value.is_a?(Array) && value.size == 1 && value.first.is_a?(String))
-          error "should be a string or an array with one string (for secret lookup)"
+            error "should be a string or an array with one string (for secret lookup)"
+          end
         end
       end
     end

test/cli/main_test.rb

@@ -22,7 +22,7 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
     # deploy
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -46,7 +46,7 @@ class CliMainTest < CliTestCase
     with_test_secrets("secrets" => "DB_PASSWORD=secret") do
       invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true }
 
-      Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
+      Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: false))
       Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
       Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
       Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -72,7 +72,7 @@ class CliMainTest < CliTestCase
   test "deploy with skip_push" do
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -159,7 +159,7 @@ class CliMainTest < CliTestCase
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, :skip_local => false }
 
     Kamal::Cli::Main.any_instance.expects(:invoke)
-      .with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
+      .with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: false))
       .raises(RuntimeError)
 
     assert_not KAMAL.holding_lock?
@@ -172,7 +172,7 @@ class CliMainTest < CliTestCase
   test "deploy with skipped hooks" do
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => true }
 
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: false))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -187,7 +187,7 @@ class CliMainTest < CliTestCase
   test "deploy with missing secrets" do
     invoke_options = { "config_file" => "test/fixtures/deploy_with_secrets.yml", "version" => "999", "skip_hooks" => false }
 
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:setup", [], invoke_options.merge(skip_local: false))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
@@ -289,6 +289,16 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "remove" do
+    options = { "config_file" => "test/fixtures/deploy_simple.yml", "skip_hooks" => false, "confirmed" => true }
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:remove", [], options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:remove", [], options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:remove", [ "all" ], options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:remove", [], options.merge(skip_local: true))
+
+    run_command("remove", "-y")
+  end
+
   test "details" do
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:details")
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:details")

test/cli/registry_test.rb

@@ -1,50 +1,62 @@
 require_relative "cli_test_case"
 
 class CliRegistryTest < CliTestCase
-  test "login" do
+  test "setup" do
-    run_command("login").tap do |output|
+    run_command("setup").tap do |output|
       assert_match /docker login -u \[REDACTED\] -p \[REDACTED\] as .*@localhost/, output
       assert_match /docker login -u \[REDACTED\] -p \[REDACTED\] on 1.1.1.\d/, output
     end
   end
 
-  test "login skip local" do
+  test "setup skip local" do
-    run_command("login", "-L").tap do |output|
+    run_command("setup", "-L").tap do |output|
       assert_no_match /docker login -u \[REDACTED\] -p \[REDACTED\] as .*@localhost/, output
       assert_match /docker login -u \[REDACTED\] -p \[REDACTED\] on 1.1.1.\d/, output
     end
   end
 
-  test "login skip remote" do
+  test "setup skip remote" do
-    run_command("login", "-R").tap do |output|
+    run_command("setup", "-R").tap do |output|
       assert_match /docker login -u \[REDACTED\] -p \[REDACTED\] as .*@localhost/, output
       assert_no_match /docker login -u \[REDACTED\] -p \[REDACTED\] on 1.1.1.\d/, output
     end
   end
 
-  test "logout" do
+  test "remove" do
-    run_command("logout").tap do |output|
+    run_command("remove").tap do |output|
       assert_match /docker logout as .*@localhost/, output
       assert_match /docker logout on 1.1.1.\d/, output
     end
   end
 
-  test "logout skip local" do
+  test "remove skip local" do
-    run_command("logout", "-L").tap do |output|
+    run_command("remove", "-L").tap do |output|
       assert_no_match /docker logout as .*@localhost/, output
       assert_match /docker logout on 1.1.1.\d/, output
     end
   end
 
-  test "logout skip remote" do
+  test "remove skip remote" do
-    run_command("logout", "-R").tap do |output|
+    run_command("remove", "-R").tap do |output|
       assert_match /docker logout as .*@localhost/, output
       assert_no_match /docker logout on 1.1.1.\d/, output
     end
   end
 
+  test "setup local registry" do
+    run_command("setup", fixture: :with_local_registry).tap do |output|
+      assert_match /docker start kamal-docker-registry || docker run --detach -p 5000:5000 --name kamal-docker-registry registry:2 as .*@localhost/, output
+    end
+  end
+
+  test "remove local registry" do
+    run_command("remove", fixture: :with_local_registry).tap do |output|
+      assert_match /docker stop kamal-docker-registry && docker rm kamal-docker-registry as .*@localhost/, output
+    end
+  end
+
   private
-    def run_command(*command)
+    def run_command(*command, fixture: :with_accessories)
-      stdouted { Kamal::Cli::Registry.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }
+      stdouted { Kamal::Cli::Registry.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) }
     end
 end

test/commands/registry_test.rb

@@ -55,6 +55,14 @@ class CommandsRegistryTest < ActiveSupport::TestCase
       registry.logout.join(" ")
   end
 
+  test "registry setup" do
+    assert_equal "docker start kamal-docker-registry || docker run --detach -p :5000 --name kamal-docker-registry registry:2", registry.setup.join(" ")
+  end
+
+  test "registry remove" do
+    assert_equal "docker stop kamal-docker-registry && docker rm kamal-docker-registry", registry.remove.join(" ")
+  end
+
   private
     def registry
       Kamal::Commands::Registry.new Kamal::Configuration.new(@config)

test/integration/app_test.rb

@@ -27,14 +27,14 @@ class AppTest < IntegrationTest
     images = kamal :app, :images, capture: true
     assert_match "App Host: vm1", images
     assert_match "App Host: vm2", images
-    assert_match /registry:4443\/app\s+#{latest_app_version}/, images
+    assert_match /localhost:5000\/app\s+#{latest_app_version}/, images
-    assert_match /registry:4443\/app\s+latest/, images
+    assert_match /localhost:5000\/app\s+latest/, images
 
     containers = kamal :app, :containers, capture: true
     assert_match "App Host: vm1", containers
     assert_match "App Host: vm2", containers
-    assert_match "registry:4443/app:#{latest_app_version}", containers
+    assert_match "localhost:5000/app:#{latest_app_version}", containers
-    assert_match "registry:4443/app:latest", containers
+    assert_match "localhost:5000/app:latest", containers
 
     exec_output = kamal :app, :exec, :ps, capture: true
     assert_match "App Host: vm1", exec_output

test/integration/docker/deployer/app/config/deploy.yml

@@ -26,9 +26,7 @@ readiness_delay: 0
 proxy:
   host: 127.0.0.1
 registry:
-  server: registry:4443
+  server: localhost:5000
-  username: root
-  password: root
 builder:
   driver: docker
   arch: <%= Kamal::Utils.docker_arch %>

test/integration/main_test.rb

@@ -29,7 +29,7 @@ class MainTest < IntegrationTest
     assert_match /App Host: vm1/, details
     assert_match /App Host: vm2/, details
     assert_match /basecamp\/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION}/, details
-    assert_match /registry:4443\/app:#{first_version}/, details
+    assert_match /localhost:5000\/app:#{first_version}/, details
 
     audit = kamal :audit, capture: true
     assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit
@@ -63,8 +63,8 @@ class MainTest < IntegrationTest
     assert_equal [ "vm1", "vm2" ], config[:hosts]
     assert_equal "vm1", config[:primary_host]
     assert_equal version, config[:version]
-    assert_equal "registry:4443/app", config[:repository]
+    assert_equal "localhost:5000/app", config[:repository]
-    assert_equal "registry:4443/app:#{version}", config[:absolute_image]
+    assert_equal "localhost:5000/app:#{version}", config[:absolute_image]
     assert_equal "app-#{version}", config[:service_with_version]
     assert_equal [], config[:volume_args]
     assert_equal({ user: "root", port: 22, keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])