Bump version for 1.9.3

v3 was recently released which broke the integration tests. Update them
to use the correct config file.

Set the major version to prevent this from happening when v4 is
released.

.github/workflows/ci.yml

@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - main
+      - 1-9-stable
   pull_request:
 jobs:
   rubocop:
@@ -24,25 +25,15 @@ jobs:
     strategy:
       matrix:
         ruby-version:
-          - "2.7"
           - "3.1"
           - "3.2"
           - "3.3"
         gemfile:
           - Gemfile
-          - gemfiles/ruby_2.7.gemfile
           - gemfiles/rails_edge.gemfile
         exclude:
-          - ruby-version: "2.7"
-            gemfile: Gemfile
-          - ruby-version: "2.7"
-            gemfile: gemfiles/rails_edge.gemfile
           - ruby-version: "3.1"
-            gemfile: gemfiles/ruby_2.7.gemfile
+            gemfile: gemfiles/rails_edge.gemfile
-          - ruby-version: "3.2"
-            gemfile: gemfiles/ruby_2.7.gemfile
-          - ruby-version: "3.3"
-            gemfile: gemfiles/ruby_2.7.gemfile
     name: ${{ format('Tests (Ruby {0})', matrix.ruby-version) }}
     runs-on: ubuntu-latest
     continue-on-error: true

.github/workflows/docker-publish.yml

@@ -6,7 +6,7 @@ on:
       tagInput:
         description: 'Tag'
         required: true
-    
+
   release:
     types: [created]
     tags:
@@ -51,5 +51,4 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: |
-            ghcr.io/basecamp/kamal:latest
             ghcr.io/basecamp/kamal:${{ steps.version-tag.outputs.value }}

Dockerfile

@@ -1,7 +1,7 @@
 # Use the official Ruby 3.2.0 Alpine image as the base image
 FROM ruby:3.2.0-alpine
 
-# Install  docker/buildx-bin
+# Install docker/buildx-bin
 COPY --from=docker/buildx-bin /buildx /usr/libexec/docker/cli-plugins/docker-buildx
 
 # Set the working directory to /kamal
@@ -14,7 +14,7 @@ COPY Gemfile Gemfile.lock kamal.gemspec ./
 COPY lib/kamal/version.rb /kamal/lib/kamal/version.rb
 
 # Install system dependencies
-RUN apk add --no-cache --update build-base git docker openrc openssh-client-default \
+RUN apk add --no-cache build-base git docker openrc openssh-client-default \
     && rc-update add docker boot \
     && gem install bundler --version=2.4.3 \
     && bundle install
@@ -33,7 +33,7 @@ WORKDIR /workdir
 
 # Tell git it's safe to access /workdir/.git even if
 # the directory is owned by a different user
-RUN git config --global --add safe.directory /workdir
+RUN git config --global --add safe.directory '*'
 
 # Set the entrypoint to run the installed binary in /workdir
 # Example:  docker run -it -v "$PWD:/workdir" kamal init

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kamal (1.7.0)
+    kamal (1.9.3)
       activesupport (>= 7.0)
       base64 (~> 0.2)
       bcrypt_pbkdf (~> 1.0)
@@ -9,9 +9,8 @@ PATH
       dotenv (~> 2.8)
       ed25519 (~> 1.2)
       net-ssh (~> 7.0)
-      sshkit (>= 1.22.2, < 2.0)
+      sshkit (>= 1.23.0, < 2.0)
       thor (~> 1.2)
-      x25519 (~> 1.0, >= 1.0.10)
       zeitwerk (~> 2.5)
 
 GEM
@@ -79,11 +78,11 @@ GEM
     net-sftp (4.0.0)
       net-ssh (>= 5.0.0, < 8.0.0)
     net-ssh (7.2.1)
-    nokogiri (1.16.0-arm64-darwin)
+    nokogiri (1.18.8-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.16.0-x86_64-darwin)
+    nokogiri (1.18.8-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.16.0-x86_64-linux)
+    nokogiri (1.18.8-x86_64-linux-gnu)
       racc (~> 1.4)
     parallel (1.24.0)
     parser (3.3.0.5)
@@ -154,9 +153,8 @@ GEM
       rubocop-rails
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
-    sshkit (1.22.2)
+    sshkit (1.23.0)
       base64
-      mutex_m
       net-scp (>= 1.1.2)
       net-sftp (>= 2.1.2)
       net-ssh (>= 2.8.0)
@@ -166,7 +164,6 @@ GEM
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.5.0)
     webrick (1.8.1)
-    x25519 (1.0.10)
     zeitwerk (2.6.12)
 
 PLATFORMS

gemfiles/ruby_2.7.gemfile

@@ -1,6 +0,0 @@
-source 'https://rubygems.org'
-git_source(:github) { |repo| "https://github.com/#{repo}.git" }
-
-gemspec path: "../"
-
-gem "nokogiri", "~> 1.15.0"

kamal.gemspec

@@ -12,13 +12,12 @@ Gem::Specification.new do |spec|
   spec.executables = %w[ kamal ]
 
   spec.add_dependency "activesupport", ">= 7.0"
-  spec.add_dependency "sshkit", ">= 1.22.2", "< 2.0"
+  spec.add_dependency "sshkit", ">= 1.23.0", "< 2.0"
   spec.add_dependency "net-ssh", "~> 7.0"
   spec.add_dependency "thor", "~> 1.2"
   spec.add_dependency "dotenv", "~> 2.8"
   spec.add_dependency "zeitwerk", "~> 2.5"
   spec.add_dependency "ed25519", "~> 1.2"
-  spec.add_dependency "x25519", "~> 1.0", ">= 1.0.10"
   spec.add_dependency "bcrypt_pbkdf", "~> 1.0"
   spec.add_dependency "concurrent-ruby", "~> 1.2"
   spec.add_dependency "base64", "~> 0.2"

lib/kamal/cli/accessory.rb

@@ -222,6 +222,25 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     end
   end
 
+  desc "downgrade", "Downgrade accessories from Kamal 2 to 1.9"
+  option :rolling, type: :boolean, default: false, desc: "Upgrade one host at a time"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  def downgrade(name)
+    confirming "This will restart all accessories" do
+      with_lock do
+        host_groups = options[:rolling] ? KAMAL.accessory_hosts : [ KAMAL.accessory_hosts ]
+        host_groups.each do |hosts|
+          host_list = Array(hosts).join(",")
+          KAMAL.with_specific_hosts(hosts) do
+            say "Downgrading #{name} accessories on #{host_list}...", :magenta
+            reboot name
+            say "Downgraded #{name} accessories on #{host_list}...", :magenta
+          end
+        end
+      end
+    end
+  end
+
   private
     def with_accessory(name)
       if KAMAL.config.accessory(name)

lib/kamal/cli/app.rb

@@ -14,7 +14,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
         end
 
         # Primary hosts and roles are returned first, so they can open the barrier
-        barrier = Kamal::Cli::Healthcheck::Barrier.new if KAMAL.roles.many?
+        barrier = Kamal::Cli::Healthcheck::Barrier.new
 
         on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
           KAMAL.roles_on(host).each do |role|

lib/kamal/cli/app/boot.rb

@@ -72,7 +72,7 @@ class Kamal::Cli::App::Boot
 
     def release_barrier
       if barrier.open
-        info "First #{KAMAL.primary_role} container is healthy on #{host}, booting other roles"
+        info "First #{KAMAL.primary_role} container is healthy on #{host}, booting any other roles"
       end
     end
 
@@ -87,7 +87,7 @@ class Kamal::Cli::App::Boot
 
     def close_barrier
       if barrier.close
-        info "First #{KAMAL.primary_role} container is unhealthy on #{host}, not booting other roles"
+        info "First #{KAMAL.primary_role} container is unhealthy on #{host}, not booting any other roles"
         error capture_with_info(*app.logs(version: version))
         error capture_with_info(*app.container_health_log(version: version))
       end

lib/kamal/cli/base.rb

@@ -25,23 +25,45 @@ module Kamal::Cli
     def initialize(*)
       super
       @original_env = ENV.to_h.dup
-      load_envs
+      load_env
       initialize_commander(options_with_subcommand_class_options)
     end
 
     private
-      def load_envs
+      def reload_env
+        reset_env
+        load_env
+      end
+
+      def load_env
         if destination = options[:destination]
-          Dotenv.load(".env.#{destination}", ".env")
+          Dotenv.overload(".env", ".env.#{destination}")
         else
-          Dotenv.load(".env")
+          Dotenv.overload(".env")
         end
       end
 
-      def reload_envs
+      def reset_env
+        replace_env @original_env
+      end
+
+      def replace_env(env)
         ENV.clear
-        ENV.update(@original_env)
+        ENV.update(env)
-        load_envs
+      end
+
+      def with_original_env
+        keeping_current_env do
+          reset_env
+          yield
+        end
+      end
+
+      def keeping_current_env
+        current_env = ENV.to_h.dup
+        yield
+      ensure
+        replace_env(current_env)
       end
 
       def options_with_subcommand_class_options
@@ -184,6 +206,10 @@ module Kamal::Cli
         instance_variable_get("@_invocations").first
       end
 
+      def reset_invocation(cli_class)
+        instance_variable_get("@_invocations")[cli_class].pop
+      end
+
       def ensure_run_and_locks_directory
         on(KAMAL.hosts) do
           execute(*KAMAL.server.ensure_run_directory)

lib/kamal/cli/build.rb

@@ -43,8 +43,8 @@ class Kamal::Cli::Build < Kamal::Cli::Base
           cli.create
         end
       rescue SSHKit::Command::Failed => e
-        warn "Missing compatible builder, so creating a new one first"
+        if e.message =~ /(context not found|no builder|does not exist)/
-        if e.message =~ /(context not found|no builder)/
+          warn "Missing compatible builder, so creating a new one first"
           cli.create
         else
           raise
@@ -59,11 +59,14 @@ class Kamal::Cli::Build < Kamal::Cli::Base
 
   desc "pull", "Pull app image from registry onto servers"
   def pull
-    on(KAMAL.hosts) do
+    if (first_hosts = mirror_hosts).any?
-      execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
+      #  Pull on a single host per mirror first to seed them
-      execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
+      say "Pulling image on #{first_hosts.join(", ")} to seed the #{"mirror".pluralize(first_hosts.count)}...", :magenta
-      execute *KAMAL.builder.pull
+      pull_on_hosts(first_hosts)
-      execute *KAMAL.builder.validate_image
+      say "Pulling image on remaining hosts...", :magenta
+      pull_on_hosts(KAMAL.hosts - first_hosts)
+    else
+      pull_on_hosts(KAMAL.hosts)
     end
   end
 
@@ -131,4 +134,28 @@ class Kamal::Cli::Build < Kamal::Cli::Base
         end
       end
     end
+
+    def mirror_hosts
+      if KAMAL.hosts.many?
+        mirror_hosts = Concurrent::Hash.new
+        on(KAMAL.hosts) do |host|
+          first_mirror = capture_with_info(*KAMAL.builder.first_mirror).strip.presence
+          mirror_hosts[first_mirror] ||= host.to_s if first_mirror
+        rescue SSHKit::Command::Failed => e
+          raise unless e.message =~ /error calling index: reflect: slice index out of range/
+        end
+        mirror_hosts.values
+      else
+        []
+      end
+    end
+
+    def pull_on_hosts(hosts)
+      on(hosts) do
+        execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
+        execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
+        execute *KAMAL.builder.pull
+        execute *KAMAL.builder.validate_image
+      end
+    end
 end

lib/kamal/cli/main.rb

@@ -11,6 +11,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
 
         say "Evaluate and push env files...", :magenta
         invoke "kamal:cli:main:envify", [], invoke_options
+        invoke "kamal:cli:env:push", [], invoke_options
 
         invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options
         deploy
@@ -190,10 +191,12 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     end
 
     if Pathname.new(File.expand_path(env_template_path)).exist?
-      File.write(env_path, ERB.new(File.read(env_template_path), trim_mode: "-").result, perm: 0600)
+      # Ensure existing env doesn't pollute template evaluation
+      content = with_original_env { ERB.new(File.read(env_template_path), trim_mode: "-").result }
+      File.write(env_path, content, perm: 0600)
 
       unless options[:skip_push]
-        reload_envs
+        reload_env
         invoke "kamal:cli:env:push", options
       end
     else
@@ -214,6 +217,37 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     end
   end
 
+  desc "downgrade", "Downgrade from Kamal 2 to 1.9"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  option :rolling, type: :boolean, default: false, desc: "Downgrade one host at a time"
+  def downgrade
+    confirming "This will replace Traefik with kamal-proxy and restart all accessories" do
+      with_lock do
+        if options[:rolling]
+          (KAMAL.hosts | KAMAL.accessory_hosts).each do |host|
+            KAMAL.with_specific_hosts(host) do
+              say "Downgrading #{host}...", :magenta
+              if KAMAL.hosts.include?(host)
+                invoke "kamal:cli:traefik:downgrade", [], options.merge(confirmed: true, rolling: false)
+                reset_invocation(Kamal::Cli::Traefik)
+              end
+              if KAMAL.accessory_hosts.include?(host)
+                invoke "kamal:cli:accessory:downgrade", [ "all" ], options.merge(confirmed: true, rolling: false)
+                reset_invocation(Kamal::Cli::Accessory)
+              end
+              say "Downgraded #{host}", :magenta
+            end
+          end
+        else
+          say "Downgrading all hosts...", :magenta
+          invoke "kamal:cli:traefik:downgrade", [], options.merge(confirmed: true)
+          invoke "kamal:cli:accessory:downgrade", [ "all" ], options.merge(confirmed: true)
+          say "Downgraded all hosts", :magenta
+        end
+      end
+    end
+  end
+
   desc "version", "Show Kamal version"
   def version
     puts Kamal::VERSION

lib/kamal/cli/templates/sample_hooks/docker-setup.sample

@@ -1,7 +1,13 @@
-#!/bin/sh
+#!/usr/bin/env ruby
 
 # A sample docker-setup hook
 #
-# Sets up a Docker network which can then be used by the application’s containers
+# Sets up a Docker network on defined hosts which can then be used by the application’s containers
 
-ssh user@example.com docker network create kamal
+hosts = ENV["KAMAL_HOSTS"].split(",")
+
+hosts.each do |ip|
+  destination = "root@#{ip}"
+  puts "Creating a Docker network \"kamal\" on #{destination}"
+  `ssh #{destination} docker network create kamal`
+end

lib/kamal/cli/traefik.rb

@@ -119,4 +119,44 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
       end
     end
   end
+
+  desc "downgrade", "Downgrade to Traefik on servers (stop container, remove container, start new container, reboot app)"
+  option :rolling, type: :boolean, default: false, desc: "Reboot proxy on hosts in sequence, rather than in parallel"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  def downgrade
+    invoke_options = { "version" => KAMAL.config.latest_tag }.merge(options)
+
+    confirming "This will cause a brief outage on each host. Are you sure?" do
+      host_groups = options[:rolling] ? KAMAL.hosts : [ KAMAL.hosts ]
+      host_groups.each do |hosts|
+        host_list = Array(hosts).join(",")
+        say "Downgrading to Traefik on #{host_list}...", :magenta
+        run_hook "pre-traefik-reboot", hosts: host_list
+        on(hosts) do |host|
+          execute *KAMAL.auditor.record("Rebooted Traefik"), verbosity: :debug
+          execute *KAMAL.registry.login
+
+          "Stopping and removing kamal-proxy on #{host}, if running..."
+          execute *KAMAL.traefik.cleanup_kamal_proxy
+
+          "Stopping and removing Traefik on #{host}, if running..."
+          execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
+          execute *KAMAL.traefik.remove_container
+          execute *KAMAL.traefik.remove_image
+        end
+
+        KAMAL.with_specific_hosts(hosts) do
+          invoke "kamal:cli:traefik:boot", [], invoke_options
+          reset_invocation(Kamal::Cli::Traefik)
+          invoke "kamal:cli:app:boot", [], invoke_options
+          reset_invocation(Kamal::Cli::App)
+          invoke "kamal:cli:prune:all", [], invoke_options
+          reset_invocation(Kamal::Cli::Prune)
+        end
+
+        run_hook "post-traefik-reboot", hosts: host_list
+        say "Downgraded to Traefik on #{host_list}", :magenta
+      end
+    end
+  end
 end

lib/kamal/commander.rb

@@ -56,6 +56,13 @@ class Kamal::Commander
     end
   end
 
+  def with_specific_hosts(hosts)
+    original_hosts, self.specific_hosts = specific_hosts, hosts
+    yield
+  ensure
+    self.specific_hosts = original_hosts
+  end
+
   def accessory_names
     config.accessories&.collect(&:name) || []
   end

lib/kamal/commander/specifics.rb

@@ -23,7 +23,7 @@ class Kamal::Commander::Specifics
   end
 
   def accessory_hosts
-    specific_hosts || config.accessories.flat_map(&:hosts)
+    config.accessories.flat_map(&:hosts) & specified_hosts
   end
 
   private

lib/kamal/commands/auditor.rb

@@ -9,7 +9,7 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
   # Runs remotely
   def record(line, **details)
     append \
-      [ :echo, audit_tags(**details).except(:version, :service_version).to_s, line ],
+      [ :echo, audit_tags(**details).except(:version, :service_version, :service).to_s, line ],
       audit_log_file
   end
 

lib/kamal/commands/builder.rb

@@ -2,7 +2,7 @@ require "active_support/core_ext/string/filters"
 
 class Kamal::Commands::Builder < Kamal::Commands::Base
   delegate :create, :remove, :push, :clean, :pull, :info, :context_hosts, :config_context_hosts, :validate_image,
-    to: :target
+           :first_mirror, to: :target
 
   include Clone
 

lib/kamal/commands/builder/base.rb

@@ -40,6 +40,10 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
     []
   end
 
+  def first_mirror
+    docker(:info, "--format '{{index .RegistryConfig.Mirrors 0}}'")
+  end
+
   private
     def build_tags
       [ "-t", config.absolute_image, "-t", config.latest_image ]

lib/kamal/commands/builder/clone.rb

@@ -6,7 +6,7 @@ module Kamal::Commands::Builder::Clone
   end
 
   def clone
-    git :clone, Kamal::Git.root, path: clone_directory
+    git :clone, Kamal::Git.root, "--recurse-submodules", path: clone_directory
   end
 
   def clone_reset_steps
@@ -14,7 +14,8 @@ module Kamal::Commands::Builder::Clone
       git(:remote, "set-url", :origin, Kamal::Git.root, path: build_directory),
       git(:fetch, :origin, path: build_directory),
       git(:reset, "--hard", Kamal::Git.revision, path: build_directory),
-      git(:clean, "-fdx", path: build_directory)
+      git(:clean, "-fdx", path: build_directory),
+      git(:submodule, :update, "--init", path: build_directory)
     ]
   end
 

lib/kamal/commands/builder/multiarch/remote.rb

@@ -58,4 +58,8 @@ class Kamal::Commands::Builder::Multiarch::Remote < Kamal::Commands::Builder::Mu
     def remove_context(arch)
       docker :context, :rm, builder_name_with_arch(arch)
     end
+
+    def platform_names
+      "linux/#{local_arch},linux/#{remote_arch}"
+    end
 end

lib/kamal/commands/traefik.rb

@@ -62,6 +62,15 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
     [ :rm, "-f", env.secrets_file ]
   end
 
+  def cleanup_kamal_proxy
+    chain \
+      docker(:container, :stop, "kamal-proxy"),
+      combine(
+        docker(:container, :prune, "--force", "--filter", "label=org.opencontainers.image.title=kamal-proxy"),
+        docker(:image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=kamal-proxy")
+      )
+  end
+
   private
     def publish_args
       argumentize "--publish", port if publish?

lib/kamal/configuration.rb

@@ -47,7 +47,7 @@ class Kamal::Configuration
     @destination = destination
     @declared_version = version
 
-    validate! raw_config, example: validation_yml.symbolize_keys, context: ""
+    validate! raw_config, example: validation_yml.symbolize_keys, context: "", with: Kamal::Configuration::Validator::Configuration
 
     # Eager load config to validate it, these are first as they have dependencies later on
     @servers = Servers.new(config: self)

lib/kamal/configuration/docs/configuration.yml

@@ -2,13 +2,24 @@
 #
 # Configuration is read from the `config/deploy.yml`
 #
+
+# Destinations
+#
 # When running commands, you can specify a destination with the `-d` flag,
 # e.g. `kamal deploy -d staging`
 #
 # In this case the configuration will also be read from `config/deploy.staging.yml`
 # and merged with the base configuration.
+
+# Extensions
 #
-# The available configuration options are explained below.
+# Kamal will not accept unrecognized keys in the configuration file.
+#
+# However, you might want to declare a configuration block using YAML anchors
+# and aliases to avoid repetition.
+#
+# You can use prefix a configuration section with `x-` to indicate that it is an
+# extension. Kamal will ignore the extension and not raise an error.
 
 # The service name
 # This is a required value. It is used as the container name prefix.

lib/kamal/configuration/docs/env.yml

@@ -29,7 +29,7 @@ env:
 # To pass the secrets you should list them under the `secret` key. When you do this the
 # other variables need to be moved under the `clear` key.
 #
-# Unlike clear valies, secrets are not passed directly to the container,
+# Unlike clear values, secrets are not passed directly to the container,
 # but are stored in an env file on the host
 # The file is not updated when deploying, only when running `kamal envify` or `kamal env push`.
 env:

lib/kamal/configuration/docs/ssh.yml

@@ -44,3 +44,23 @@ ssh:
   # Defaults to `fatal`. Set this to debug if you are having
   # SSH connection issues.
   log_level: debug
+
+  # Keys Only
+  #
+  # Set to true to use only private keys from keys and key_data parameters, 
+  # even if ssh-agent offers more identities. This option is intended for 
+  # situations where ssh-agent offers many different identites or you have 
+  # a need to overwrite all identites and force a single one.
+  keys_only: false
+
+  # Keys
+  #
+  # An array of file names of private keys to use for publickey
+  # and hostbased authentication
+  keys: [ "~/.ssh/id.pem" ]
+
+  # Key Data
+  #
+  # An array of strings, with each element of the array being
+  # a raw private key in PEM format.
+  key_data: [ "-----BEGIN OPENSSH PRIVATE KEY-----" ]

lib/kamal/configuration/docs/traefik.yml

@@ -17,8 +17,8 @@ traefik:
 
   # Image
   #
-  # The Traefik image to use, defaults to `traefik:v2.10`
+  # The Traefik image to use, defaults to `traefik:v2.11`
-  image: traefik:v2.9
+  image: traefik:v2.11
 
   # Host port
   #

lib/kamal/configuration/ssh.rb

@@ -26,8 +26,20 @@ class Kamal::Configuration::Ssh
     end
   end
 
+  def keys_only
+    ssh_config["keys_only"]
+  end
+
+  def keys
+    ssh_config["keys"]
+  end
+
+  def key_data
+    ssh_config["key_data"]
+  end
+
   def options
-    { user: user, port: port, proxy: proxy, logger: logger, keepalive: true, keepalive_interval: 30 }.compact
+    { user: user, port: port, proxy: proxy, logger: logger, keepalive: true, keepalive_interval: 30, keys_only: keys_only, keys: keys, key_data: key_data }.compact
   end
 
   def to_h

lib/kamal/configuration/traefik.rb

@@ -1,5 +1,5 @@
 class Kamal::Configuration::Traefik
-  DEFAULT_IMAGE = "traefik:v2.10"
+  DEFAULT_IMAGE = "traefik:v2.11"
   CONTAINER_PORT = 80
   DEFAULT_ARGS = {
     "log.level" => "DEBUG"

lib/kamal/configuration/validator.rb

@@ -15,11 +15,10 @@ class Kamal::Configuration::Validator
     def validate_against_example!(validation_config, example)
       validate_type! validation_config, Hash
 
-      if (unknown_keys = validation_config.keys - example.keys).any?
+      check_unknown_keys! validation_config, example
-        unknown_keys_error unknown_keys
-      end
 
       validation_config.each do |key, value|
+        next if extension?(key)
         with_context(key) do
           example_value = example[key]
 
@@ -31,9 +30,9 @@ class Kamal::Configuration::Validator
             validate_array_of! value, example_value.first.class
           elsif example_value.is_a?(Hash)
             case key.to_s
-            when "options"
+            when "options", "args"
               validate_type! value, Hash
-            when "args", "labels"
+            when "labels"
               validate_hash_of! value, example_value.first[1].class
             else
               validate_against_example! value, example_value
@@ -137,4 +136,18 @@ class Kamal::Configuration::Validator
     ensure
       @context = old_context
     end
+
+    def allow_extensions?
+      false
+    end
+
+    def extension?(key)
+      key.to_s.start_with?("x-")
+    end
+
+    def check_unknown_keys!(config, example)
+      unknown_keys = config.keys - example.keys
+      unknown_keys.reject! { |key| extension?(key) } if allow_extensions?
+      unknown_keys_error unknown_keys if unknown_keys.present?
+    end
 end

lib/kamal/configuration/validator/configuration.rb

@@ -0,0 +1,6 @@
+class Kamal::Configuration::Validator::Configuration < Kamal::Configuration::Validator
+  private
+    def allow_extensions?
+      true
+    end
+end

lib/kamal/git.rb

@@ -9,6 +9,10 @@ module Kamal::Git
     `git config user.name`.strip
   end
 
+  def email
+    `git config user.email`.strip
+  end
+
   def revision
     `git rev-parse HEAD`.strip
   end

lib/kamal/tags.rb

@@ -10,10 +10,11 @@ class Kamal::Tags
 
     def default_tags(config)
       { recorded_at: Time.now.utc.iso8601,
-        performer: `whoami`.chomp,
+        performer: Kamal::Git.email.presence || `whoami`.chomp,
         destination: config.destination,
         version: config.version,
-        service_version: service_version(config) }
+        service_version: service_version(config),
+        service: config.service }
     end
 
     def service_version(config)

lib/kamal/version.rb

@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "1.7.0"
+  VERSION = "1.9.3"
 end

test/cli/accessory_test.rb

@@ -209,6 +209,24 @@ class CliAccessoryTest < CliTestCase
     end
   end
 
+  test "downgrade" do
+    run_command("downgrade", "-y", "all").tap do |output|
+      assert_match "Downgrading all accessories on 1.1.1.3,1.1.1.1,1.1.1.2...", output
+      assert_match "docker container stop app-mysql on 1.1.1.3", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "Downgraded all accessories on 1.1.1.3,1.1.1.1,1.1.1.2", output
+    end
+  end
+
+  test "downgrade rolling" do
+    run_command("downgrade", "--rolling", "-y", "all").tap do |output|
+      assert_match "Downgrading all accessories on 1.1.1.3...", output
+      assert_match "docker container stop app-mysql on 1.1.1.3", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "Downgraded all accessories on 1.1.1.3", output
+    end
+  end
+
   private
     def run_command(*command)
       stdouted { Kamal::Cli::Accessory.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }

test/cli/build_test.rb

@@ -42,7 +42,7 @@ class CliBuildTest < CliTestCase
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
-        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd)
+        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd, "--recurse-submodules")
         .raises(SSHKit::Command::Failed.new("fatal: destination path 'kamal' already exists and is not an empty directory"))
         .then
         .returns(true)
@@ -50,6 +50,7 @@ class CliBuildTest < CliTestCase
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :fetch, :origin)
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :reset, "--hard", Kamal::Git.revision)
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :clean, "-fdx")
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :submodule, :update, "--init")
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
         .with(:docker, :buildx, :build, "--push", "--platform", "linux/amd64,linux/arm64", "--builder", "kamal-app-multiarch", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".")
@@ -88,7 +89,7 @@ class CliBuildTest < CliTestCase
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
-        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd)
+        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd, "--recurse-submodules")
         .raises(SSHKit::Command::Failed.new("fatal: destination path 'kamal' already exists and is not an empty directory"))
         .then
         .returns(true)
@@ -169,12 +170,41 @@ class CliBuildTest < CliTestCase
 
   test "pull" do
     run_command("pull").tap do |output|
+      assert_match /docker info --format '{{index .RegistryConfig.Mirrors 0}}'/, output
       assert_match /docker image rm --force dhh\/app:999/, output
       assert_match /docker pull dhh\/app:999/, output
       assert_match "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:999 | grep -x app || (echo \"Image dhh/app:999 is missing the 'service' label\" && exit 1)", output
     end
   end
 
+  test "pull with mirror" do
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :info, "--format '{{index .RegistryConfig.Mirrors 0}}'")
+      .returns("registry-mirror.example.com")
+      .at_least_once
+
+    run_command("pull").tap do |output|
+      assert_match /Pulling image on 1\.1\.1\.\d to seed the mirror\.\.\./, output
+      assert_match "Pulling image on remaining hosts...", output
+      assert_equal 4, output.scan(/docker pull dhh\/app:999/).size, output
+      assert_match "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:999 | grep -x app || (echo \"Image dhh/app:999 is missing the 'service' label\" && exit 1)", output
+    end
+  end
+
+  test "pull with mirrors" do
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :info, "--format '{{index .RegistryConfig.Mirrors 0}}'")
+      .returns("registry-mirror.example.com", "registry-mirror2.example.com")
+      .at_least_once
+
+    run_command("pull").tap do |output|
+      assert_match /Pulling image on 1\.1\.1\.\d, 1\.1\.1\.\d to seed the mirrors\.\.\./, output
+      assert_match "Pulling image on remaining hosts...", output
+      assert_equal 4, output.scan(/docker pull dhh\/app:999/).size, output
+      assert_match "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:999 | grep -x app || (echo \"Image dhh/app:999 is missing the 'service' label\" && exit 1)", output
+    end
+  end
+
   test "create" do
     run_command("create").tap do |output|
       assert_match /docker buildx create --use --name kamal-app-multiarch/, output

test/cli/cli_test_case.rb

@@ -42,16 +42,19 @@ class CliTestCase < ActiveSupport::TestCase
     end
 
     def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: false)
-      performer = `whoami`.strip
+      whoami = `whoami`.chomp
+      performer = Kamal::Git.email.presence || whoami
+      service = service_version.split("@").first
 
       assert_match "Running the #{hook} hook...\n", output
 
-      expected = %r{Running\s/usr/bin/env\s\.kamal/hooks/#{hook}\sas\s#{performer}@localhost\n\s
+      expected = %r{Running\s/usr/bin/env\s\.kamal/hooks/#{hook}\sas\s#{whoami}@localhost\n\s
         DEBUG\s\[[0-9a-f]*\]\sCommand:\s\(\sexport\s
         KAMAL_RECORDED_AT=\"\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ\"\s
         KAMAL_PERFORMER=\"#{performer}\"\s
         KAMAL_VERSION=\"#{version}\"\s
         KAMAL_SERVICE_VERSION=\"#{service_version}\"\s
+        KAMAL_SERVICE=\"#{service}\"\s
         KAMAL_HOSTS=\"#{hosts}\"\s
         KAMAL_COMMAND=\"#{command}\"\s
         #{"KAMAL_SUBCOMMAND=\\\"#{subcommand}\\\"\\s" if subcommand}

test/cli/main_test.rb

@@ -1,11 +1,15 @@
 require_relative "cli_test_case"
 
 class CliMainTest < CliTestCase
+  setup { @original_env = ENV.to_h.dup }
+  teardown { ENV.clear; ENV.update @original_env }
+
   test "setup" do
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:main:envify", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:deploy)
 
@@ -19,6 +23,7 @@ class CliMainTest < CliTestCase
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:main:envify", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options)
     # deploy
@@ -120,6 +125,11 @@ class CliMainTest < CliTestCase
       .with(:docker, :buildx, :inspect, "kamal-app-multiarch", "> /dev/null")
       .returns("")
 
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :info, "--format '{{index .RegistryConfig.Mirrors 0}}'")
+      .returns("")
+      .at_least_once
+
     assert_raises(Kamal::Cli::LockError) do
       run_command("deploy")
     end
@@ -153,6 +163,11 @@ class CliMainTest < CliTestCase
       .with(:docker, :buildx, :inspect, "kamal-app-multiarch", "> /dev/null")
       .returns("")
 
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :info, "--format '{{index .RegistryConfig.Mirrors 0}}'")
+      .returns("")
+      .at_least_once
+
     assert_raises(SSHKit::Runner::ExecuteError) do
       run_command("deploy")
     end
@@ -432,7 +447,7 @@ class CliMainTest < CliTestCase
   end
 
   test "envify" do
-    with_test_dot_env_erb(contents: "HELLO=<%= 'world' %>") do
+    with_test_dotenv(".env.erb": "HELLO=<%= 'world' %>") do
       run_command("envify")
       assert_equal("HELLO=world", File.read(".env"))
     end
@@ -446,14 +461,14 @@ class CliMainTest < CliTestCase
       <% end  -%>
     EOF
 
-    with_test_dot_env_erb(contents: file) do
+    with_test_dotenv(".env.erb": file) do
       run_command("envify")
       assert_equal("HELLO=world\nKEY=value\n", File.read(".env"))
     end
   end
 
   test "envify with destination" do
-    with_test_dot_env_erb(contents: "HELLO=<%= 'world' %>", file: ".env.world.erb") do
+    with_test_dotenv(".env.world.erb": "HELLO=<%= 'world' %>") do
       run_command("envify", "-d", "world", config_file: "deploy_for_dest")
       assert_equal "HELLO=world", File.read(".env.world")
     end
@@ -468,6 +483,46 @@ class CliMainTest < CliTestCase
     run_command("envify", "--skip-push")
   end
 
+  test "envify with clean env" do
+    with_test_dotenv(".env": "HELLO=already", ".env.erb": "HELLO=<%= ENV.fetch 'HELLO', 'never' %>") do
+      run_command("envify", "--skip-push")
+      assert_equal "HELLO=never", File.read(".env")
+    end
+  end
+
+  test "env files overwrite shell environment variables" do
+    ENV["TEST_VAR"] = "shell_value"
+    ENV["AWS_ACCESS_KEY_ID"] = "local_dev_key"
+
+    with_test_dotenv(".env": "TEST_VAR=dotenv_value\nAWS_ACCESS_KEY_ID=production_key") do
+      # Create a simple CLI command instance to trigger load_env
+      Kamal::Cli::Main.new.send(:load_env)
+
+      assert_equal "dotenv_value", ENV["TEST_VAR"]
+      assert_equal "production_key", ENV["AWS_ACCESS_KEY_ID"]
+    end
+  ensure
+    ENV.delete("TEST_VAR")
+    ENV.delete("AWS_ACCESS_KEY_ID")
+  end
+
+  test "destination env files overwrite base env files" do
+    ENV["TEST_VAR"] = "shell_value"
+
+    with_test_dotenv(".env": "TEST_VAR=base_value\nBASE_ONLY=base", ".env.world": "TEST_VAR=world_value\nWORLD_ONLY=world") do
+      # Create CLI command with destination to trigger load_env
+      Kamal::Cli::Main.new([], { destination: "world" }).send(:load_env)
+
+      assert_equal "world_value", ENV["TEST_VAR"]
+      assert_equal "base", ENV["BASE_ONLY"]
+      assert_equal "world", ENV["WORLD_ONLY"]
+    end
+  ensure
+    ENV.delete("TEST_VAR")
+    ENV.delete("BASE_ONLY")
+    ENV.delete("WORLD_ONLY")
+  end
+
   test "remove with confirmation" do
     run_command("remove", "-y", config_file: "deploy_with_accessories").tap do |output|
       assert_match /docker container stop traefik/, output
@@ -515,19 +570,49 @@ class CliMainTest < CliTestCase
     assert_equal Kamal::VERSION, version
   end
 
+  test "downgrade" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_with_accessories.yml", "skip_hooks" => false, "confirmed" => true, "rolling" => false }
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:downgrade", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:downgrade", [ "all" ], invoke_options)
+
+    run_command("downgrade", "-y", config_file: "deploy_with_accessories").tap do |output|
+      assert_match "Downgrading all hosts...", output
+      assert_match "Downgraded all hosts", output
+    end
+  end
+
+  test "downgrade rolling" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_with_accessories.yml", "skip_hooks" => false, "confirmed" => true, "rolling" => false }
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:downgrade", [], invoke_options).times(4)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:downgrade", [ "all" ], invoke_options).times(3)
+
+    run_command("downgrade", "--rolling", "-y", config_file: "deploy_with_accessories").tap do |output|
+      assert_match "Downgrading 1.1.1.1...", output
+      assert_match "Downgraded 1.1.1.1", output
+      assert_match "Downgrading 1.1.1.2...", output
+      assert_match "Downgraded 1.1.1.2", output
+      assert_match "Downgrading 1.1.1.3...", output
+      assert_match "Downgraded 1.1.1.3", output
+      assert_match "Downgrading 1.1.1.4...", output
+      assert_match "Downgraded 1.1.1.4", output
+    end
+  end
+
   private
     def run_command(*command, config_file: "deploy_simple")
       stdouted { Kamal::Cli::Main.start([ *command, "-c", "test/fixtures/#{config_file}.yml" ]) }
     end
 
-    def with_test_dot_env_erb(contents:, file: ".env.erb")
+    def with_test_dotenv(**files)
       Dir.mktmpdir do |dir|
         fixtures_dup = File.join(dir, "test")
         FileUtils.mkdir_p(fixtures_dup)
         FileUtils.cp_r("test/fixtures/", fixtures_dup)
 
         Dir.chdir(dir) do
-          File.write(file, contents)
+          files.each do |filename, contents|
+            File.binwrite(filename.to_s, contents)
+          end
           yield
         end
       end

test/cli/traefik_test.rb

@@ -103,6 +103,90 @@ class CliTraefikTest < CliTestCase
     end
   end
 
+  test "downgrade" do
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678")
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678")
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with { |*args| args[0..1] == [ :sh, "-c" ] }
+      .returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running").at_least_once # workers health check
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
+      .returns("") # old version
+
+    run_command("downgrade", "-y").tap do |output|
+      assert_match "Downgrading to Traefik on 1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4...", output
+      assert_match "docker login -u [REDACTED] -p [REDACTED]", output
+      assert_match "docker container stop kamal-proxy ; docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy && docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy", output
+      assert_match "docker container stop traefik", output
+      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik", output
+      assert_match "docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik", output
+      assert_match "/usr/bin/env mkdir -p .kamal", output
+      assert_match "docker login -u [REDACTED] -p [REDACTED]", output
+      assert_match "docker container start traefik || docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" traefik:v2.11 --providers.docker --log.level=\"DEBUG\"", output
+      assert_match "/usr/bin/env mkdir -p .kamal", output
+      assert_match %r{docker rename app-web-latest app-web-latest_replaced_.*}, output
+      assert_match %r{docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-.* -e KAMAL_CONTAINER_NAME="app-web-latest" -e KAMAL_VERSION="latest" --env-file .kamal/env/roles/app-web.env --health-cmd}, output
+      assert_match "docker tag dhh/app:latest dhh/app:latest", output
+      assert_match "/usr/bin/env mkdir -p .kamal", output
+      assert_match "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done", output
+      assert_match "docker image prune --force --filter label=service=app", output
+      assert_match "Downgraded to Traefik on 1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", output
+    end
+  end
+
+  test "downgrade rolling" do
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678")
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678")
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with { |*args| args[0..1] == [ :sh, "-c" ] }
+      .returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running").at_least_once # workers health check
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
+      .returns("") # old version
+
+    run_command("downgrade", "--rolling", "-y",).tap do |output|
+      %w[1.1.1.1 1.1.1.2 1.1.1.3 1.1.1.4].each do |host|
+        assert_match "Downgrading to Traefik on #{host}...", output
+        assert_match "docker container stop kamal-proxy ; docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy && docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy", output
+        assert_match "Downgraded to Traefik on #{host}", output
+      end
+    end
+  end
+
   private
     def run_command(*command)
       stdouted { Kamal::Cli::Traefik.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }

test/commands/auditor_test.rb

@@ -12,7 +12,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase
     }
 
     @auditor = new_command
-    @performer = `whoami`.strip
+    @performer = Kamal::Git.email.presence || `whoami`.chomp
     @recorded_at = Time.now.utc.iso8601
   end
 

test/commands/builder_test.rb

@@ -30,10 +30,10 @@ class CommandsBuilderTest < ActiveSupport::TestCase
   end
 
   test "target multiarch remote when local and remote is set" do
-    builder = new_builder_command(builder: { "local" => {}, "remote" => {}, "cache" => { "type" => "gha" } })
+    builder = new_builder_command(builder: { "local" => { "arch" => "arm64" }, "remote" => { "arch" => "amd64" }, "cache" => { "type" => "gha" } })
     assert_equal "multiarch/remote", builder.name
     assert_equal \
-      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --push --platform linux/arm64,linux/amd64 --builder kamal-app-multiarch-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
       builder.push.join(" ")
   end
 
@@ -200,6 +200,11 @@ class CommandsBuilderTest < ActiveSupport::TestCase
     assert_equal [ "unix:///var/run/docker.sock", "ssh://host" ], command.config_context_hosts
   end
 
+  test "mirror count" do
+    command = new_builder_command
+    assert_equal "docker info --format '{{index .RegistryConfig.Mirrors 0}}'", command.first_mirror.join(" ")
+  end
+
   private
     def new_builder_command(additional_config = {})
       Kamal::Commands::Builder.new(Kamal::Configuration.new(@config.merge(additional_config), version: "123"))

test/commands/hook_test.rb

@@ -11,7 +11,7 @@ class CommandsHookTest < ActiveSupport::TestCase
       traefik: { "args" => { "accesslog.format" => "json", "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } }
     }
 
-    @performer = `whoami`.strip
+    @performer = Kamal::Git.email.presence || `whoami`.chomp
     @recorded_at = Time.now.utc.iso8601
   end
 
@@ -22,7 +22,8 @@ class CommandsHookTest < ActiveSupport::TestCase
         "KAMAL_RECORDED_AT" => @recorded_at,
         "KAMAL_PERFORMER" => @performer,
         "KAMAL_VERSION" => "123",
-        "KAMAL_SERVICE_VERSION" => "app@123" } }
+        "KAMAL_SERVICE_VERSION" => "app@123",
+        "KAMAL_SERVICE" => "app" } }
     ], new_command.run("foo")
   end
 
@@ -33,7 +34,8 @@ class CommandsHookTest < ActiveSupport::TestCase
         "KAMAL_RECORDED_AT" => @recorded_at,
         "KAMAL_PERFORMER" => @performer,
         "KAMAL_VERSION" => "123",
-        "KAMAL_SERVICE_VERSION" => "app@123" } }
+        "KAMAL_SERVICE_VERSION" => "app@123",
+        "KAMAL_SERVICE" => "app" } }
     ], new_command(hooks_path: "custom/hooks/path").run("foo")
   end
 

test/commands/traefik_test.rb

@@ -111,6 +111,11 @@ class CommandsTraefikTest < ActiveSupport::TestCase
       new_command.run.join(" ")
   end
 
+  test "run with args array" do
+    @config[:traefik]["args"] = { "entrypoints.web.forwardedheaders.trustedips" => %w[ 127.0.0.1 127.0.0.2 ] }
+    assert_equal "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" traefik:test --providers.docker --log.level=\"DEBUG\" --entrypoints.web.forwardedheaders.trustedips=\"127.0.0.1\" --entrypoints.web.forwardedheaders.trustedips=\"127.0.0.2\"", new_command.run.join(" ")
+  end
+
   test "traefik start" do
     assert_equal \
       "docker container start traefik",

test/configuration/validation_test.rb

@@ -94,7 +94,7 @@ class ConfigurationValidationTest < ActiveSupport::TestCase
     assert_error "builder/remote: unknown key: foo", builder: { "remote" => { "foo" => "bar" } }
     assert_error "builder/local: unknown key: foo", builder: { "local" => { "foo" => "bar" } }
     assert_error "builder/remote/arch: should be a string", builder: { "remote" => { "arch" => [] } }
-    assert_error "builder/args/foo: should be a string", builder: { "args" => { "foo" => [] } }
+    assert_error "builder/args: should be a hash", builder: { "args" => [ "foo" ] }
     assert_error "builder/cache/options: should be a string", builder: { "cache" => { "options" => [] } }
   end
 

test/configuration_test.rb

@@ -344,4 +344,12 @@ class ConfigurationTest < ActiveSupport::TestCase
 
     assert_raises(Kamal::ConfigurationError) { Kamal::Configuration.new(@deploy_with_roles.merge(retain_containers: 0)) }
   end
+
+  test "extensions" do
+    dest_config_file = Pathname.new(File.expand_path("fixtures/deploy_with_extensions.yml", __dir__))
+
+    config = Kamal::Configuration.create_from config_file: dest_config_file
+    assert_equal config.role(:web_tokyo).running_traefik?, true
+    assert_equal config.role(:web_chicago).running_traefik?, true
+  end
 end

test/fixtures/deploy_with_extensions.yml

@@ -0,0 +1,24 @@
+
+x-web: &web
+  traefik: true
+
+service: app
+image: dhh/app
+servers:
+  web_chicago:
+    <<: *web
+    hosts:
+      - 1.1.1.1
+      - 1.1.1.2
+  web_tokyo:
+    <<: *web
+    hosts:
+      - 1.1.1.3
+      - 1.1.1.4
+env:
+  REDIS_URL: redis://x/y
+registry:
+  server: registry.digitalocean.com
+  username: user
+  password: pw
+primary_role: web_tokyo

test/integration/broken_deploy_test.rb

@@ -26,7 +26,7 @@ class BrokenDeployTest < IntegrationTest
   private
     def assert_failed_deploy(output)
       assert_match "Waiting for the first healthy web container before booting workers on vm3...", output
-      assert_match /First web container is unhealthy on vm[12], not booting other roles/, output
+      assert_match /First web container is unhealthy on vm[12], not booting any other roles/, output
       assert_match "First web container is unhealthy, not booting workers on vm3", output
       assert_match "nginx: [emerg] unexpected end of file, expecting \";\" or \"}\" in /etc/nginx/conf.d/default.conf:2", output
       assert_match 'ERROR {"Status":"unhealthy","FailingStreak":0,"Log":[]}', output

test/integration/docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.7"
 name: "kamal-test"
 
 volumes:
@@ -30,8 +29,6 @@ services:
       context: docker/registry
     environment:
       - REGISTRY_HTTP_ADDR=0.0.0.0:4443
-      - REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt
-      - REGISTRY_HTTP_TLS_KEY=/certs/domain.key
     volumes:
       - shared:/shared
       - registry:/var/lib/registry/

test/integration/docker/deployer/Dockerfile

@@ -22,7 +22,6 @@ COPY app_with_roles/ app_with_roles/
 
 RUN rm -rf /root/.ssh
 RUN ln -s /shared/ssh /root/.ssh
-RUN mkdir -p /etc/docker/certs.d/registry:4443 && ln -s /shared/certs/domain.crt /etc/docker/certs.d/registry:4443/ca.crt
 
 RUN git config --global user.email "deployer@example.com"
 RUN git config --global user.name "Deployer"

test/integration/docker/deployer/app/config/deploy.yml

@@ -33,7 +33,7 @@ traefik:
   args:
     accesslog: true
     accesslog.format: json
-  image: registry:4443/traefik:v2.10
+  image: registry:4443/traefik:v2.11
 accessories:
   busybox:
     service: custom-busybox

test/integration/docker/deployer/app_with_roles/config/deploy.yml

@@ -27,7 +27,7 @@ traefik:
   args:
     accesslog: true
     accesslog.format: json
-  image: registry:4443/traefik:v2.10
+  image: registry:4443/traefik:v2.11
 accessories:
   busybox:
     service: custom-busybox

test/integration/docker/deployer/boot.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-dockerd --max-concurrent-downloads 1 &
+dockerd --max-concurrent-downloads 1 --insecure-registry registry:4443 &
 
 exec sleep infinity

test/integration/docker/deployer/setup.sh

@@ -19,7 +19,7 @@ push_image_to_registry_4443() {
 
 install_kamal
 push_image_to_registry_4443 nginx 1-alpine-slim
-push_image_to_registry_4443 traefik v2.10
+push_image_to_registry_4443 traefik v2.11
 push_image_to_registry_4443 busybox 1.36.0
 
 # .ssh is on a shared volume that persists between runs. Clean it up as the

test/integration/docker/registry/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry
+FROM registry:3
 
 COPY boot.sh .
 

test/integration/docker/registry/boot.sh

@@ -1,5 +1,3 @@
 #!/bin/sh
 
-while [ ! -f /certs/domain.crt ]; do sleep 1; done
+exec /entrypoint.sh /etc/distribution/config.yml
-
-exec /entrypoint.sh /etc/docker/registry/config.yml

test/integration/docker/shared/Dockerfile

@@ -10,8 +10,6 @@ RUN mkdir ssh && \
 COPY registry-dns.conf .
 COPY boot.sh .
 
-RUN mkdir certs && openssl req -newkey rsa:4096 -nodes -sha256 -keyout certs/domain.key   -x509 -days 365 -out certs/domain.crt   -subj '/CN=registry' -extensions EXT -config registry-dns.conf
-
 HEALTHCHECK --interval=1s CMD pgrep sleep
 
 CMD ["./boot.sh"]

test/integration/docker/vm/Dockerfile

@@ -5,7 +5,6 @@ WORKDIR /work
 RUN apt-get update --fix-missing && apt-get -y install openssh-client openssh-server docker.io
 
 RUN mkdir /root/.ssh && ln -s /shared/ssh/id_rsa.pub /root/.ssh/authorized_keys
-RUN mkdir -p /etc/docker/certs.d/registry:4443 && ln -s /shared/certs/domain.crt /etc/docker/certs.d/registry:4443/ca.crt
 
 RUN echo "HOST_TOKEN=abcd" >> /etc/environment
 

test/integration/docker/vm/boot.sh

@@ -4,6 +4,6 @@ while [ ! -f /root/.ssh/authorized_keys ]; do echo "Waiting for ssh keys"; sleep
 
 service ssh restart
 
-dockerd --max-concurrent-downloads 1 &
+dockerd --max-concurrent-downloads 1 --insecure-registry registry:4443 &
 
 exec sleep infinity

test/integration/main_test.rb

@@ -32,7 +32,7 @@ class MainTest < IntegrationTest
     assert_match /Traefik Host: vm2/, details
     assert_match /App Host: vm1/, details
     assert_match /App Host: vm2/, details
-    assert_match /traefik:v2.10/, details
+    assert_match /traefik:v2.11/, details
     assert_match /registry:4443\/app:#{first_version}/, details
 
     audit = kamal :audit, capture: true

test/integration/traefik_test.rb

@@ -52,11 +52,11 @@ class TraefikTest < IntegrationTest
 
   private
     def assert_traefik_running
-      assert_match /traefik:v2.10   "\/entrypoint.sh/, traefik_details
+      assert_match /traefik:v2.11   "\/entrypoint.sh/, traefik_details
     end
 
     def assert_traefik_not_running
-      assert_no_match /traefik:v2.10   "\/entrypoint.sh/, traefik_details
+      assert_no_match /traefik:v2.11   "\/entrypoint.sh/, traefik_details
     end
 
     def traefik_details