Concurrent local and remote docker login

Login to the local and remote docker registries concurrently to shave
a couple of seconds off a deployment.

.github/workflows/ci.yml

@@ -1,5 +1,5 @@
 name: CI
-on: 
+on:
   push:
     branches:
       - main
@@ -12,17 +12,29 @@ jobs:
           - "2.7"
           - "3.1"
           - "3.2"
+          - "3.3"
         gemfile:
           - Gemfile
+          - gemfiles/ruby_2.7.gemfile
           - gemfiles/rails_edge.gemfile
-        continue-on-error: [false]
+        exclude:
+          - ruby-version: "2.7"
+            gemfile: Gemfile
+          - ruby-version: "2.7"
+            gemfile: gemfiles/rails_edge.gemfile
+          - ruby-version: "3.1"
+            gemfile: gemfiles/ruby_2.7.gemfile
+          - ruby-version: "3.2"
+            gemfile: gemfiles/ruby_2.7.gemfile
+          - ruby-version: "3.3"
+            gemfile: gemfiles/ruby_2.7.gemfile
     name: ${{ format('Tests (Ruby {0})', matrix.ruby-version) }}
     runs-on: ubuntu-latest
-    continue-on-error: ${{ matrix.continue-on-error }}
+    continue-on-error: true
     env:
       BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install Ruby
         uses: ruby/setup-ruby@v1

Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    kamal (1.2.0)
+    kamal (1.3.1)
       activesupport (>= 7.0)
+      base64 (~> 0.2)
       bcrypt_pbkdf (~> 1.0)
       concurrent-ruby (~> 1.2)
       dotenv (~> 2.8)
@@ -15,82 +16,111 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.0.4.3)
-      actionview (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      rack (~> 2.0, >= 2.2.0)
+    actionpack (7.1.2)
+      actionview (= 7.1.2)
+      activesupport (= 7.1.2)
+      nokogiri (>= 1.8.5)
+      racc
+      rack (>= 2.2.4)
+      rack-session (>= 1.0.1)
       rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (7.0.4.3)
-      activesupport (= 7.0.4.3)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+    actionview (7.1.2)
+      activesupport (= 7.1.2)
       builder (~> 3.1)
-      erubi (~> 1.4)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activesupport (7.0.4.3)
+      erubi (~> 1.11)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+    activesupport (7.1.2)
+      base64
+      bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
+      mutex_m
       tzinfo (~> 2.0)
+    base64 (0.2.0)
     bcrypt_pbkdf (1.1.0)
+    bigdecimal (3.1.5)
     builder (3.2.4)
     concurrent-ruby (1.2.2)
+    connection_pool (2.4.1)
     crass (1.0.6)
-    debug (1.7.2)
-      irb (>= 1.5.0)
-      reline (>= 0.3.1)
+    debug (1.9.1)
+      irb (~> 1.10)
+      reline (>= 0.3.8)
     dotenv (2.8.1)
+    drb (2.2.0)
+      ruby2_keywords
     ed25519 (1.3.0)
     erubi (1.12.0)
-    i18n (1.12.0)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    io-console (0.6.0)
-    irb (1.6.3)
-      reline (>= 0.3.0)
-    loofah (2.20.0)
+    io-console (0.7.1)
+    irb (1.11.0)
+      rdoc
+      reline (>= 0.3.8)
+    loofah (2.22.0)
       crass (~> 1.0.2)
-      nokogiri (>= 1.5.9)
-    method_source (1.0.0)
-    minitest (5.18.0)
-    mocha (2.0.2)
+      nokogiri (>= 1.12.0)
+    minitest (5.20.0)
+    mocha (2.1.0)
       ruby2_keywords (>= 0.0.5)
+    mutex_m (0.2.0)
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
-    net-ssh (7.1.0)
-    nokogiri (1.14.2-arm64-darwin)
+    net-ssh (7.2.1)
+    nokogiri (1.16.0-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.2-x86_64-darwin)
+    nokogiri (1.16.0-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.2-x86_64-linux)
+    nokogiri (1.16.0-x86_64-linux)
       racc (~> 1.4)
-    racc (1.6.2)
-    rack (2.2.6.4)
+    psych (5.1.2)
+      stringio
+    racc (1.7.3)
+    rack (3.0.8)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
     rack-test (2.1.0)
       rack (>= 1.3)
-    rails-dom-testing (2.0.3)
-      activesupport (>= 4.2.0)
+    rackup (2.1.0)
+      rack (>= 3)
+      webrick (~> 1.8)
+    rails-dom-testing (2.2.0)
+      activesupport (>= 5.0.0)
+      minitest
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.5.0)
-      loofah (~> 2.19, >= 2.19.1)
-    railties (7.0.4.3)
-      actionpack (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      method_source
+    rails-html-sanitizer (1.6.0)
+      loofah (~> 2.21)
+      nokogiri (~> 1.14)
+    railties (7.1.2)
+      actionpack (= 7.1.2)
+      activesupport (= 7.1.2)
+      irb
+      rackup (>= 1.0.0)
       rake (>= 12.2)
-      thor (~> 1.0)
-      zeitwerk (~> 2.5)
-    rake (13.0.6)
-    reline (0.3.3)
+      thor (~> 1.0, >= 1.2.2)
+      zeitwerk (~> 2.6)
+    rake (13.1.0)
+    rdoc (6.6.2)
+      psych (>= 4.0.0)
+    reline (0.4.2)
       io-console (~> 0.5)
     ruby2_keywords (0.0.5)
-    sshkit (1.21.4)
+    sshkit (1.21.7)
+      mutex_m
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
-    thor (1.2.1)
+    stringio (3.1.0)
+    thor (1.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.7)
+    webrick (1.8.1)
+    zeitwerk (2.6.12)
 
 PLATFORMS
   arm64-darwin

gemfiles/ruby_2.7.gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+gemspec path: "../"
+
+gem "nokogiri", "~> 1.15.0"

kamal.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "ed25519", "~> 1.2"
   spec.add_dependency "bcrypt_pbkdf", "~> 1.0"
   spec.add_dependency "concurrent-ruby", "~> 1.2"
+  spec.add_dependency "base64", "~> 0.2"
 
   spec.add_development_dependency "debug"
   spec.add_development_dependency "mocha"

lib/kamal/cli/registry.rb

@@ -1,8 +1,7 @@
 class Kamal::Cli::Registry < Kamal::Cli::Base
   desc "login", "Log in to registry locally and remotely"
   def login
-    run_locally    { execute *KAMAL.registry.login }
-    on(KAMAL.hosts) { execute *KAMAL.registry.login }
+    on([ :local ] + KAMAL.hosts) { execute *KAMAL.registry.login }
   # FIXME: This rescue needed?
   rescue ArgumentError => e
     puts e.message

lib/kamal/commands/builder/multiarch.rb

@@ -10,7 +10,7 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
   def push
     docker :buildx, :build,
       "--push",
-      "--platform", "linux/amd64,linux/arm64",
+      "--platform", platform_names,
       "--builder", builder_name,
       *build_options,
       build_context
@@ -26,4 +26,12 @@ class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
     def builder_name
       "kamal-#{config.service}-multiarch"
     end
+
+    def platform_names
+      if local_arch
+        "linux/#{local_arch}"
+      else
+        "linux/amd64,linux/arm64"
+      end
+    end
 end

lib/kamal/commands/lock.rb

@@ -1,5 +1,6 @@
 require "active_support/duration"
 require "time"
+require "base64"
 
 class Kamal::Commands::Lock < Kamal::Commands::Base
   def acquire(message, version)

lib/kamal/configuration/accessory.rb

@@ -70,8 +70,8 @@ class Kamal::Configuration::Accessory
 
   def directories
     specifics["directories"]&.to_h do |host_to_container_mapping|
-      host_relative_path, container_path = host_to_container_mapping.split(":")
-      [ expand_host_path(host_relative_path), container_path ]
+      host_path, container_path = host_to_container_mapping.split(":")
+      [ expand_host_path(host_path), container_path ]
     end || {}
   end
 
@@ -138,13 +138,17 @@ class Kamal::Configuration::Accessory
 
     def remote_directories_as_volumes
       specifics["directories"]&.collect do |host_to_container_mapping|
-        host_relative_path, container_path = host_to_container_mapping.split(":")
-        [ expand_host_path(host_relative_path), container_path ].join(":")
+        host_path, container_path = host_to_container_mapping.split(":")
+        [ expand_host_path(host_path), container_path ].join(":")
       end || []
     end
 
-    def expand_host_path(host_relative_path)
-      "#{service_data_directory}/#{host_relative_path}"
+    def expand_host_path(host_path)
+      absolute_path?(host_path) ? host_path : "#{service_data_directory}/#{host_path}"
+    end
+
+    def absolute_path?(path)
+      Pathname.new(path).absolute?
     end
 
     def service_data_directory

lib/kamal/configuration/role.rb

@@ -239,7 +239,7 @@ class Kamal::Configuration::Role
         clear_app_env  = config.env["secret"] ? Array(config.env["clear"]) : Array(config.env["clear"] || config.env)
         clear_role_env = specialized_env["secret"] ? Array(specialized_env["clear"]) : Array(specialized_env["clear"] || specialized_env)
 
-        new_env["clear"] = (clear_app_env + clear_role_env).uniq
+        new_env["clear"] = clear_app_env.to_h.merge(clear_role_env.to_h)
       end
     end
 

lib/kamal/version.rb

@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "1.2.0"
+  VERSION = "1.3.1"
 end

test/commands/builder_test.rb

@@ -37,6 +37,14 @@ class CommandsBuilderTest < ActiveSupport::TestCase
       builder.push.join(" ")
   end
 
+  test "target multiarch local when arch is set" do
+    builder = new_builder_command(builder: { "local" => { "arch" => "amd64" } })
+    assert_equal "multiarch", builder.name
+    assert_equal \
+      "docker buildx build --push --platform linux/amd64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .",
+      builder.push.join(" ")
+  end
+
   test "target native remote when only remote is set" do
     builder = new_builder_command(builder: { "remote" => { "arch" => "amd64" }, "cache" => { "type" => "gha" } })
     assert_equal "native/remote", builder.name

test/configuration/accessory_test.rb

@@ -149,10 +149,16 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
     assert_match "%", @config.accessory(:mysql).files.keys[2].read
   end
 
-  test "directories" do
+  test "directory with a relative path" do
+    @deploy[:accessories]["mysql"]["directories"] = [ "data:/var/lib/mysql" ]
     assert_equal({"$PWD/app-mysql/data"=>"/var/lib/mysql"}, @config.accessory(:mysql).directories)
   end
 
+  test "directory with an absolute path" do
+    @deploy[:accessories]["mysql"]["directories"] = [ "/var/data/mysql:/var/lib/mysql" ]
+    assert_equal({"/var/data/mysql"=>"/var/lib/mysql"}, @config.accessory(:mysql).directories)
+  end
+
   test "options" do
     assert_equal ["--cpus", "\"4\"", "--memory", "\"2GB\""], @config.accessory(:redis).option_args
   end

test/configuration/role_test.rb

@@ -176,6 +176,34 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     ENV["REDIS_PASSWORD"] = nil
   end
 
+  test "env overwritten by role with secrets" do
+    @deploy_with_roles[:env] = {
+      "clear" => {
+        "REDIS_URL" => "redis://a/b"
+      },
+      "secret" => [
+        "REDIS_PASSWORD"
+      ]
+    }
+
+    @deploy_with_roles[:servers]["workers"]["env"] = {
+      "clear" => {
+        "REDIS_URL" => "redis://c/d",
+      },
+    }
+
+    ENV["REDIS_PASSWORD"] = "secret456"
+
+    expected = <<~ENV
+      REDIS_PASSWORD=secret456
+      REDIS_URL=redis://c/d
+    ENV
+
+    assert_equal expected, @config_with_roles.role(:workers).env_file.to_s
+  ensure
+    ENV["REDIS_PASSWORD"] = nil
+  end
+
   test "host_env_directory" do
     assert_equal ".kamal/env/roles", @config_with_roles.role(:workers).host_env_directory
   end

test/integration/docker/deployer/setup.sh

@@ -21,3 +21,7 @@ install_kamal
 push_image_to_registry_4443 nginx 1-alpine-slim
 push_image_to_registry_4443 traefik v2.9
 push_image_to_registry_4443 busybox 1.36.0
+
+# .ssh is on a shared volume that persists between runs. Clean it up as the
+# churn of temporary vm IPs can eventually create conflicts.
+rm -f /root/.ssh/known_hosts

test/integration/integration_test.rb

@@ -103,7 +103,7 @@ class IntegrationTest < ActiveSupport::TestCase
       assert_equal "200", code
     end
 
-    def wait_for_healthy(timeout: 20)
+    def wait_for_healthy(timeout: 30)
       timeout_at = Time.now + timeout
       while docker_compose("ps -a | tail -n +2 | grep -v '(healthy)' | wc -l", capture: true) != "0"
         if timeout_at < Time.now