Bump version for 1.9.3

v3 was recently released which broke the integration tests. Update them
to use the correct config file.

Set the major version to prevent this from happening when v4 is
released.

.github/workflows/ci.yml

@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - main
+      - 1-9-stable
   pull_request:
 jobs:
   rubocop:
@@ -30,6 +31,9 @@ jobs:
         gemfile:
           - Gemfile
           - gemfiles/rails_edge.gemfile
+        exclude:
+          - ruby-version: "3.1"
+            gemfile: gemfiles/rails_edge.gemfile
     name: ${{ format('Tests (Ruby {0})', matrix.ruby-version) }}
     runs-on: ubuntu-latest
     continue-on-error: true

.github/workflows/docker-publish.yml

@@ -51,5 +51,4 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: |
-            ghcr.io/basecamp/kamal:latest
             ghcr.io/basecamp/kamal:${{ steps.version-tag.outputs.value }}

Dockerfile

@@ -33,7 +33,7 @@ WORKDIR /workdir
 
 # Tell git it's safe to access /workdir/.git even if
 # the directory is owned by a different user
-RUN git config --global --add safe.directory /workdir
+RUN git config --global --add safe.directory '*'
 
 # Set the entrypoint to run the installed binary in /workdir
 # Example:  docker run -it -v "$PWD:/workdir" kamal init

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kamal (1.8.1)
+    kamal (1.9.3)
       activesupport (>= 7.0)
       base64 (~> 0.2)
       bcrypt_pbkdf (~> 1.0)
@@ -78,11 +78,11 @@ GEM
     net-sftp (4.0.0)
       net-ssh (>= 5.0.0, < 8.0.0)
     net-ssh (7.2.1)
-    nokogiri (1.16.0-arm64-darwin)
+    nokogiri (1.18.8-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.16.0-x86_64-darwin)
+    nokogiri (1.18.8-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.16.0-x86_64-linux)
+    nokogiri (1.18.8-x86_64-linux-gnu)
       racc (~> 1.4)
     parallel (1.24.0)
     parser (3.3.0.5)

lib/kamal/cli/accessory.rb

@@ -222,6 +222,25 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     end
   end
 
+  desc "downgrade", "Downgrade accessories from Kamal 2 to 1.9"
+  option :rolling, type: :boolean, default: false, desc: "Upgrade one host at a time"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  def downgrade(name)
+    confirming "This will restart all accessories" do
+      with_lock do
+        host_groups = options[:rolling] ? KAMAL.accessory_hosts : [ KAMAL.accessory_hosts ]
+        host_groups.each do |hosts|
+          host_list = Array(hosts).join(",")
+          KAMAL.with_specific_hosts(hosts) do
+            say "Downgrading #{name} accessories on #{host_list}...", :magenta
+            reboot name
+            say "Downgraded #{name} accessories on #{host_list}...", :magenta
+          end
+        end
+      end
+    end
+  end
+
   private
     def with_accessory(name)
       if KAMAL.config.accessory(name)

lib/kamal/cli/base.rb

@@ -37,9 +37,9 @@ module Kamal::Cli
 
       def load_env
         if destination = options[:destination]
-          Dotenv.load(".env.#{destination}", ".env")
+          Dotenv.overload(".env", ".env.#{destination}")
         else
-          Dotenv.load(".env")
+          Dotenv.overload(".env")
         end
       end
 
@@ -206,6 +206,10 @@ module Kamal::Cli
         instance_variable_get("@_invocations").first
       end
 
+      def reset_invocation(cli_class)
+        instance_variable_get("@_invocations")[cli_class].pop
+      end
+
       def ensure_run_and_locks_directory
         on(KAMAL.hosts) do
           execute(*KAMAL.server.ensure_run_directory)

lib/kamal/cli/build.rb

@@ -30,9 +30,18 @@ class Kamal::Cli::Build < Kamal::Cli::Base
       say "Building with uncommitted changes:\n #{uncommitted_changes}", :yellow
     end
 
+    # Get the command here to ensure the Dir.chdir doesn't interfere with it
+    push = KAMAL.builder.push
+
     run_locally do
       begin
-        execute *KAMAL.builder.buildx_inspect
+        context_hosts = capture_with_info(*KAMAL.builder.context_hosts).split("\n")
+
+        if context_hosts != KAMAL.builder.config_context_hosts
+          warn "Context hosts have changed, so re-creating builder, was: #{context_hosts.join(", ")}], now: #{KAMAL.builder.config_context_hosts.join(", ")}"
+          cli.remove
+          cli.create
+        end
       rescue SSHKit::Command::Failed => e
         if e.message =~ /(context not found|no builder|does not exist)/
           warn "Missing compatible builder, so creating a new one first"
@@ -42,9 +51,6 @@ class Kamal::Cli::Build < Kamal::Cli::Base
         end
       end
 
-      # Get the command here to ensure the Dir.chdir doesn't interfere with it
-      push = KAMAL.builder.push
-
       KAMAL.with_verbosity(:debug) do
         Dir.chdir(KAMAL.config.builder.build_directory) { execute *push }
       end

lib/kamal/cli/main.rb

@@ -217,6 +217,37 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     end
   end
 
+  desc "downgrade", "Downgrade from Kamal 2 to 1.9"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  option :rolling, type: :boolean, default: false, desc: "Downgrade one host at a time"
+  def downgrade
+    confirming "This will replace Traefik with kamal-proxy and restart all accessories" do
+      with_lock do
+        if options[:rolling]
+          (KAMAL.hosts | KAMAL.accessory_hosts).each do |host|
+            KAMAL.with_specific_hosts(host) do
+              say "Downgrading #{host}...", :magenta
+              if KAMAL.hosts.include?(host)
+                invoke "kamal:cli:traefik:downgrade", [], options.merge(confirmed: true, rolling: false)
+                reset_invocation(Kamal::Cli::Traefik)
+              end
+              if KAMAL.accessory_hosts.include?(host)
+                invoke "kamal:cli:accessory:downgrade", [ "all" ], options.merge(confirmed: true, rolling: false)
+                reset_invocation(Kamal::Cli::Accessory)
+              end
+              say "Downgraded #{host}", :magenta
+            end
+          end
+        else
+          say "Downgrading all hosts...", :magenta
+          invoke "kamal:cli:traefik:downgrade", [], options.merge(confirmed: true)
+          invoke "kamal:cli:accessory:downgrade", [ "all" ], options.merge(confirmed: true)
+          say "Downgraded all hosts", :magenta
+        end
+      end
+    end
+  end
+
   desc "version", "Show Kamal version"
   def version
     puts Kamal::VERSION

lib/kamal/cli/traefik.rb

@@ -119,4 +119,44 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
       end
     end
   end
+
+  desc "downgrade", "Downgrade to Traefik on servers (stop container, remove container, start new container, reboot app)"
+  option :rolling, type: :boolean, default: false, desc: "Reboot proxy on hosts in sequence, rather than in parallel"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  def downgrade
+    invoke_options = { "version" => KAMAL.config.latest_tag }.merge(options)
+
+    confirming "This will cause a brief outage on each host. Are you sure?" do
+      host_groups = options[:rolling] ? KAMAL.hosts : [ KAMAL.hosts ]
+      host_groups.each do |hosts|
+        host_list = Array(hosts).join(",")
+        say "Downgrading to Traefik on #{host_list}...", :magenta
+        run_hook "pre-traefik-reboot", hosts: host_list
+        on(hosts) do |host|
+          execute *KAMAL.auditor.record("Rebooted Traefik"), verbosity: :debug
+          execute *KAMAL.registry.login
+
+          "Stopping and removing kamal-proxy on #{host}, if running..."
+          execute *KAMAL.traefik.cleanup_kamal_proxy
+
+          "Stopping and removing Traefik on #{host}, if running..."
+          execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
+          execute *KAMAL.traefik.remove_container
+          execute *KAMAL.traefik.remove_image
+        end
+
+        KAMAL.with_specific_hosts(hosts) do
+          invoke "kamal:cli:traefik:boot", [], invoke_options
+          reset_invocation(Kamal::Cli::Traefik)
+          invoke "kamal:cli:app:boot", [], invoke_options
+          reset_invocation(Kamal::Cli::App)
+          invoke "kamal:cli:prune:all", [], invoke_options
+          reset_invocation(Kamal::Cli::Prune)
+        end
+
+        run_hook "post-traefik-reboot", hosts: host_list
+        say "Downgraded to Traefik on #{host_list}", :magenta
+      end
+    end
+  end
 end

lib/kamal/commander.rb

@@ -56,6 +56,13 @@ class Kamal::Commander
     end
   end
 
+  def with_specific_hosts(hosts)
+    original_hosts, self.specific_hosts = specific_hosts, hosts
+    yield
+  ensure
+    self.specific_hosts = original_hosts
+  end
+
   def accessory_names
     config.accessories&.collect(&:name) || []
   end

lib/kamal/commander/specifics.rb

@@ -23,7 +23,7 @@ class Kamal::Commander::Specifics
   end
 
   def accessory_hosts
-    specific_hosts || config.accessories.flat_map(&:hosts)
+    config.accessories.flat_map(&:hosts) & specified_hosts
   end
 
   private

lib/kamal/commands/builder.rb

@@ -1,8 +1,8 @@
 require "active_support/core_ext/string/filters"
 
 class Kamal::Commands::Builder < Kamal::Commands::Base
-  delegate :create, :remove, :push, :clean, :pull, :info, :buildx_inspect, :validate_image, :first_mirror, to: :target
-  delegate :multiarch?, :local?, :remote?, to: "config.builder"
+  delegate :create, :remove, :push, :clean, :pull, :info, :context_hosts, :config_context_hosts, :validate_image,
+           :first_mirror, to: :target
 
   include Clone
 
@@ -11,27 +11,43 @@ class Kamal::Commands::Builder < Kamal::Commands::Base
   end
 
   def target
-    if remote?
-      if local?
-        hybrid
+    if config.builder.multiarch?
+      if config.builder.remote?
+        if config.builder.local?
+          multiarch_remote
+        else
+          native_remote
+        end
       else
-        remote
+        multiarch
       end
     else
-      local
+      if config.builder.cached?
+        native_cached
+      else
+        native
+      end
     end
   end
 
-  def remote
-    @remote ||= Kamal::Commands::Builder::Remote.new(config)
+  def native
+    @native ||= Kamal::Commands::Builder::Native.new(config)
   end
 
-  def local
-    @local ||= Kamal::Commands::Builder::Local.new(config)
+  def native_cached
+    @native ||= Kamal::Commands::Builder::Native::Cached.new(config)
   end
 
-  def hybrid
-    @hybrid ||= Kamal::Commands::Builder::Hybrid.new(config)
+  def native_remote
+    @native ||= Kamal::Commands::Builder::Native::Remote.new(config)
+  end
+
+  def multiarch
+    @multiarch ||= Kamal::Commands::Builder::Multiarch.new(config)
+  end
+
+  def multiarch_remote
+    @multiarch_remote ||= Kamal::Commands::Builder::Multiarch::Remote.new(config)
   end
 
 

lib/kamal/commands/builder/base.rb

@@ -1,41 +1,20 @@
+
 class Kamal::Commands::Builder::Base < Kamal::Commands::Base
   class BuilderError < StandardError; end
 
   ENDPOINT_DOCKER_HOST_INSPECT = "'{{.Endpoints.docker.Host}}'"
 
   delegate :argumentize, to: Kamal::Utils
-  delegate \
-    :args, :secrets, :dockerfile, :target, :local_arch, :remote_arch, :remote_host,
-    :cache_from, :cache_to, :multiarch?, :ssh, :driver, :docker_driver?,
-    to: :builder_config
+  delegate :args, :secrets, :dockerfile, :target, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, :ssh, to: :builder_config
 
   def clean
     docker :image, :rm, "--force", config.absolute_image
   end
 
-  def push
-    docker :build,
-      "--push",
-      *platform_options,
-      *([ "--builder", builder_name ] unless docker_driver?),
-      *build_options,
-      build_context
-  end
-
   def pull
     docker :pull, config.absolute_image
   end
 
-  def info
-    combine \
-      docker(:context, :ls),
-      docker(:buildx, :ls)
-  end
-
-  def buildx_inspect
-    docker :buildx, :inspect, builder_name
-  end
-
   def build_options
     [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_target, *build_ssh ]
   end
@@ -53,6 +32,14 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
       )
   end
 
+  def context_hosts
+    :true
+  end
+
+  def config_context_hosts
+    []
+  end
+
   def first_mirror
     docker(:info, "--format '{{index .RegistryConfig.Mirrors 0}}'")
   end

lib/kamal/commands/builder/clone.rb

@@ -6,7 +6,7 @@ module Kamal::Commands::Builder::Clone
   end
 
   def clone
-    git :clone, Kamal::Git.root, path: clone_directory
+    git :clone, Kamal::Git.root, "--recurse-submodules", path: clone_directory
   end
 
   def clone_reset_steps
@@ -14,7 +14,8 @@ module Kamal::Commands::Builder::Clone
       git(:remote, "set-url", :origin, Kamal::Git.root, path: build_directory),
       git(:fetch, :origin, path: build_directory),
       git(:reset, "--hard", Kamal::Git.revision, path: build_directory),
-      git(:clean, "-fdx", path: build_directory)
+      git(:clean, "-fdx", path: build_directory),
+      git(:submodule, :update, "--init", path: build_directory)
     ]
   end
 

lib/kamal/commands/builder/hybrid.rb

@@ -1,25 +0,0 @@
-class Kamal::Commands::Builder::Hybrid < Kamal::Commands::Builder::Remote
-  def create
-    combine \
-      create_local_buildx,
-      create_remote_context,
-      append_remote_buildx
-  end
-
-  private
-    def builder_name
-      "kamal-hybrid-#{driver}-#{local_arch}-#{remote_arch}-#{remote_host.gsub(/[^a-z0-9_-]/, "-")}"
-    end
-
-    def create_local_buildx
-      docker :buildx, :create, "--name", builder_name, "--platform", "linux/#{local_arch}", "--driver=#{driver}"
-    end
-
-    def append_remote_buildx
-      docker :buildx, :create, "--append", "--name", builder_name, builder_name, "--platform", "linux/#{remote_arch}"
-    end
-
-    def platform
-      "linux/#{local_arch},linux/#{remote_arch}"
-    end
-end

lib/kamal/commands/builder/local.rb

@@ -1,24 +0,0 @@
-class Kamal::Commands::Builder::Local < Kamal::Commands::Builder::Base
-  def create
-    docker :buildx, :create, "--name", builder_name, "--driver=#{driver}" unless docker_driver?
-  end
-
-  def remove
-    docker :buildx, :rm, builder_name unless docker_driver?
-  end
-
-  private
-    def builder_name
-      "kamal-local-#{driver}"
-    end
-
-    def platform_options
-      if multiarch?
-        if local_arch
-          [ "--platform", "linux/#{local_arch}" ]
-        else
-          [ "--platform", "linux/amd64,linux/arm64" ]
-        end
-      end
-    end
-end

lib/kamal/commands/builder/multiarch.rb

@@ -0,0 +1,41 @@
+class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
+  def create
+    docker :buildx, :create, "--use", "--name", builder_name
+  end
+
+  def remove
+    docker :buildx, :rm, builder_name
+  end
+
+  def info
+    combine \
+      docker(:context, :ls),
+      docker(:buildx, :ls)
+  end
+
+  def push
+    docker :buildx, :build,
+      "--push",
+      "--platform", platform_names,
+      "--builder", builder_name,
+      *build_options,
+      build_context
+  end
+
+  def context_hosts
+    docker :buildx, :inspect, builder_name, "> /dev/null"
+  end
+
+  private
+    def builder_name
+      "kamal-#{config.service}-multiarch"
+    end
+
+    def platform_names
+      if local_arch
+        "linux/#{local_arch}"
+      else
+        "linux/amd64,linux/arm64"
+      end
+    end
+end

lib/kamal/commands/builder/multiarch/remote.rb

@@ -0,0 +1,65 @@
+class Kamal::Commands::Builder::Multiarch::Remote < Kamal::Commands::Builder::Multiarch
+  def create
+    combine \
+      create_contexts,
+      create_local_buildx,
+      append_remote_buildx
+  end
+
+  def remove
+    combine \
+      remove_contexts,
+      super
+  end
+
+  def context_hosts
+    chain \
+      context_host(builder_name_with_arch(local_arch)),
+      context_host(builder_name_with_arch(remote_arch))
+  end
+
+  def config_context_hosts
+    [ local_host, remote_host ].compact
+  end
+
+  private
+    def builder_name
+      super + "-remote"
+    end
+
+    def builder_name_with_arch(arch)
+      "#{builder_name}-#{arch}"
+    end
+
+    def create_local_buildx
+      docker :buildx, :create, "--name", builder_name, builder_name_with_arch(local_arch), "--platform", "linux/#{local_arch}"
+    end
+
+    def append_remote_buildx
+      docker :buildx, :create, "--append", "--name", builder_name, builder_name_with_arch(remote_arch), "--platform", "linux/#{remote_arch}"
+    end
+
+    def create_contexts
+      combine \
+        create_context(local_arch, local_host),
+        create_context(remote_arch, remote_host)
+    end
+
+    def create_context(arch, host)
+      docker :context, :create, builder_name_with_arch(arch), "--description", "'#{builder_name} #{arch} native host'", "--docker", "'host=#{host}'"
+    end
+
+    def remove_contexts
+      combine \
+        remove_context(local_arch),
+        remove_context(remote_arch)
+    end
+
+    def remove_context(arch)
+      docker :context, :rm, builder_name_with_arch(arch)
+    end
+
+    def platform_names
+      "linux/#{local_arch},linux/#{remote_arch}"
+    end
+end

lib/kamal/commands/builder/native.rb

@@ -0,0 +1,20 @@
+class Kamal::Commands::Builder::Native < Kamal::Commands::Builder::Base
+  def create
+    # No-op on native without cache
+  end
+
+  def remove
+    # No-op on native without cache
+  end
+
+  def info
+    # No-op on native
+  end
+
+  def push
+    combine \
+      docker(:build, *build_options, build_context),
+      docker(:push, config.absolute_image),
+      docker(:push, config.latest_image)
+  end
+end

lib/kamal/commands/builder/native/cached.rb

@@ -0,0 +1,25 @@
+class Kamal::Commands::Builder::Native::Cached < Kamal::Commands::Builder::Native
+  def create
+    docker :buildx, :create, "--name", builder_name, "--use", "--driver=docker-container"
+  end
+
+  def remove
+    docker :buildx, :rm, builder_name
+  end
+
+  def push
+    docker :buildx, :build,
+      "--push",
+      *build_options,
+      build_context
+  end
+
+  def context_hosts
+    docker :buildx, :inspect, builder_name, "> /dev/null"
+  end
+
+  private
+    def builder_name
+      "kamal-#{config.service}-native-cached"
+    end
+end

lib/kamal/commands/builder/native/remote.rb

@@ -0,0 +1,67 @@
+class Kamal::Commands::Builder::Native::Remote < Kamal::Commands::Builder::Native
+  def create
+    chain \
+      create_context,
+      create_buildx
+  end
+
+  def remove
+    chain \
+      remove_context,
+      remove_buildx
+  end
+
+  def info
+    chain \
+      docker(:context, :ls),
+      docker(:buildx, :ls)
+  end
+
+  def push
+    docker :buildx, :build,
+    "--push",
+    "--platform", platform,
+    "--builder", builder_name,
+    *build_options,
+    build_context
+  end
+
+  def context_hosts
+    context_host(builder_name_with_arch)
+  end
+
+  def config_context_hosts
+    [ remote_host ]
+  end
+
+
+  private
+    def builder_name
+      "kamal-#{config.service}-native-remote"
+    end
+
+    def builder_name_with_arch
+      "#{builder_name}-#{remote_arch}"
+    end
+
+    def platform
+      "linux/#{remote_arch}"
+    end
+
+    def create_context
+      docker :context, :create,
+        builder_name_with_arch, "--description", "'#{builder_name} #{remote_arch} native host'", "--docker", "'host=#{remote_host}'"
+    end
+
+    def remove_context
+      docker :context, :rm, builder_name_with_arch
+    end
+
+    def create_buildx
+      docker :buildx, :create, "--name", builder_name, builder_name_with_arch, "--platform", platform
+    end
+
+    def remove_buildx
+      docker :buildx, :rm, builder_name
+    end
+end

lib/kamal/commands/builder/remote.rb

@@ -1,57 +0,0 @@
-class Kamal::Commands::Builder::Remote < Kamal::Commands::Builder::Base
-  def create
-    chain \
-      create_remote_context,
-      create_buildx
-  end
-
-  def remove
-    chain \
-      remove_remote_context,
-      remove_buildx
-  end
-
-  def info
-    chain \
-      docker(:context, :ls),
-      docker(:buildx, :ls)
-  end
-
-  def push
-    docker :build,
-      "--push",
-      *platform_options,
-      "--builder", builder_name,
-      *build_options,
-      build_context
-  end
-
-  private
-    def builder_name
-      "kamal-remote-#{driver}-#{remote_arch}-#{remote_host.gsub(/[^a-z0-9_-]/, "-")}"
-    end
-
-    def create_remote_context
-      docker :context, :create, builder_name, "--description", "'#{builder_name} host'", "--docker", "'host=#{remote_host}'"
-    end
-
-    def remove_remote_context
-      docker :context, :rm, builder_name
-    end
-
-    def create_buildx
-      docker :buildx, :create, "--name", builder_name, builder_name, "--platform", platform
-    end
-
-    def remove_buildx
-      docker :buildx, :rm, builder_name
-    end
-
-    def platform_options
-      [ "--platform", platform ]
-    end
-
-    def platform
-      "linux/#{remote_arch}"
-    end
-end

lib/kamal/commands/traefik.rb

@@ -62,6 +62,15 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
     [ :rm, "-f", env.secrets_file ]
   end
 
+  def cleanup_kamal_proxy
+    chain \
+      docker(:container, :stop, "kamal-proxy"),
+      combine(
+        docker(:container, :prune, "--force", "--filter", "label=org.opencontainers.image.title=kamal-proxy"),
+        docker(:image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=kamal-proxy")
+      )
+  end
+
   private
     def publish_args
       argumentize "--publish", port if publish?

lib/kamal/configuration/builder.rb

@@ -55,14 +55,14 @@ class Kamal::Configuration::Builder
     builder_config["context"] || "."
   end
 
-  def driver
-    builder_config.fetch("driver", "docker-container")
-  end
-
   def local_arch
     builder_config["local"]["arch"] if local?
   end
 
+  def local_host
+    builder_config["local"]["host"] if local?
+  end
+
   def remote_arch
     builder_config["remote"]["arch"] if remote?
   end
@@ -114,36 +114,7 @@ class Kamal::Configuration::Builder
       end
   end
 
-  def docker_driver?
-    driver == "docker"
-  end
-
   private
-    def valid?
-      if multiarch?
-        if local?
-          raise ArgumentError, "Invalid builder configuration: local configuration, arch required" unless local_arch
-        end
-
-        if remote?
-          raise ArgumentError, "Invalid builder configuration: remote configuration, arch required" unless remote_arch
-          raise ArgumentError, "Invalid builder configuration: remote configuration, arch required" unless remote_host
-        end
-
-        if docker_driver?
-          raise ArgumentError, "Invalid builder configuration: the docker driver does not support multiarch builds"
-        end
-      else
-        raise ArgumentError, "Invalid builder configuration: multiarch must be enabled for local configuration" if local?
-        raise ArgumentError, "Invalid builder configuration: multiarch must be enabled for remote configuration" if remote?
-      end
-
-      if @options["cache"] && @options["cache"]["type"]
-        raise ArgumentError, "Invalid cache type: #{@options["cache"]["type"]}" unless [ "gha", "registry" ].include?(@options["cache"]["type"])
-        raise ArgumentError, "The docker driver does not support caching" if docker_driver?
-      end
-    end
-
     def cache_image
       builder_config["cache"]&.fetch("image", nil) || "#{image}-build-cache"
     end

lib/kamal/configuration/docs/builder.yml

@@ -1,10 +1,10 @@
 # Builder
 #
-# The builder configuration controls how the application is built with `docker build`
+# The builder configuration controls how the application is built with `docker build` or `docker buildx build`
 #
 # If no configuration is specified, Kamal will:
 # 1. Create a buildx context called `kamal-<service>-multiarch`
-# 2. Use `docker build` to build a multiarch image for linux/amd64,linux/arm64 with that context
+# 2. Use `docker buildx build` to build a multiarch image for linux/amd64,linux/arm64 with that context
 #
 # See https://kamal-deploy.org/docs/configuration/builder-examples/ for more information
 
@@ -18,11 +18,6 @@ builder:
   # Enables multiarch builds, defaults to `true`
   multiarch: false
 
-  # Driver
-  #
-  # The build driver to use, defaults to `docker-container`
-  driver: docker
-
   # Local configuration
   #
   # The build configuration for local builds, only used if multiarch is enabled (the default)

lib/kamal/configuration/docs/traefik.yml

@@ -17,8 +17,8 @@ traefik:
 
   # Image
   #
-  # The Traefik image to use, defaults to `traefik:v2.10`
-  image: traefik:v2.9
+  # The Traefik image to use, defaults to `traefik:v2.11`
+  image: traefik:v2.11
 
   # Host port
   #

lib/kamal/configuration/traefik.rb

@@ -1,5 +1,5 @@
 class Kamal::Configuration::Traefik
-  DEFAULT_IMAGE = "traefik:v2.10"
+  DEFAULT_IMAGE = "traefik:v2.11"
   CONTAINER_PORT = 80
   DEFAULT_ARGS = {
     "log.level" => "DEBUG"

lib/kamal/version.rb

@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "1.8.1"
+  VERSION = "1.9.3"
 end

test/cli/accessory_test.rb

@@ -209,6 +209,24 @@ class CliAccessoryTest < CliTestCase
     end
   end
 
+  test "downgrade" do
+    run_command("downgrade", "-y", "all").tap do |output|
+      assert_match "Downgrading all accessories on 1.1.1.3,1.1.1.1,1.1.1.2...", output
+      assert_match "docker container stop app-mysql on 1.1.1.3", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "Downgraded all accessories on 1.1.1.3,1.1.1.1,1.1.1.2", output
+    end
+  end
+
+  test "downgrade rolling" do
+    run_command("downgrade", "--rolling", "-y", "all").tap do |output|
+      assert_match "Downgrading all accessories on 1.1.1.3...", output
+      assert_match "docker container stop app-mysql on 1.1.1.3", output
+      assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output
+      assert_match "Downgraded all accessories on 1.1.1.3", output
+    end
+  end
+
   private
     def run_command(*command)
       stdouted { Kamal::Cli::Accessory.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }

test/cli/build_test.rb

@@ -21,12 +21,16 @@ class CliBuildTest < CliTestCase
         .with(:git, "-C", anything, :status, "--porcelain")
         .returns("")
 
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:docker, :buildx, :inspect, "kamal-app-multiarch", "> /dev/null")
+        .returns("")
+
       run_command("push", "--verbose").tap do |output|
         assert_hook_ran "pre-build", output, **hook_variables
         assert_match /Cloning repo into build directory/, output
         assert_match /git -C #{Dir.tmpdir}\/kamal-clones\/app-#{pwd_sha} clone #{Dir.pwd}/, output
         assert_match /docker --version && docker buildx version/, output
-        assert_match /docker build --push --platform linux\/amd64,linux\/arm64 --builder kamal-local -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
+        assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile \. as .*@localhost/, output
       end
     end
   end
@@ -38,7 +42,7 @@ class CliBuildTest < CliTestCase
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
-        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd)
+        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd, "--recurse-submodules")
         .raises(SSHKit::Command::Failed.new("fatal: destination path 'kamal' already exists and is not an empty directory"))
         .then
         .returns(true)
@@ -46,9 +50,10 @@ class CliBuildTest < CliTestCase
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :fetch, :origin)
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :reset, "--hard", Kamal::Git.revision)
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :clean, "-fdx")
+      SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :submodule, :update, "--init")
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
-        .with(:docker, :build, "--push", "--platform", "linux/amd64,linux/arm64", "--builder", "kamal-local", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".")
+        .with(:docker, :buildx, :build, "--push", "--platform", "linux/amd64,linux/arm64", "--builder", "kamal-app-multiarch", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".")
 
       SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
         .with(:git, "-C", anything, :"rev-parse", :HEAD)
@@ -73,7 +78,7 @@ class CliBuildTest < CliTestCase
       assert_no_match /Cloning repo into build directory/, output
       assert_hook_ran "pre-build", output, **hook_variables
       assert_match /docker --version && docker buildx version/, output
-      assert_match /docker build --push --platform linux\/amd64,linux\/arm64 --builder kamal-local -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile . as .*@localhost/, output
+      assert_match /docker buildx build --push --platform linux\/amd64,linux\/arm64 --builder kamal-app-multiarch -t dhh\/app:999 -t dhh\/app:latest --label service="app" --file Dockerfile . as .*@localhost/, output
     end
   end
 
@@ -84,7 +89,7 @@ class CliBuildTest < CliTestCase
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "--version", "&&", :docker, :buildx, "version")
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
-        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd)
+        .with(:git, "-C", "#{Dir.tmpdir}/kamal-clones/app-#{pwd_sha}", :clone, Dir.pwd, "--recurse-submodules")
         .raises(SSHKit::Command::Failed.new("fatal: destination path 'kamal' already exists and is not an empty directory"))
         .then
         .returns(true)
@@ -119,10 +124,10 @@ class CliBuildTest < CliTestCase
         .with(:docker, "--version", "&&", :docker, :buildx, "version")
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
-        .with(:docker, :buildx, :create, "--name", "kamal-local", "--driver=docker-container")
+        .with(:docker, :buildx, :create, "--use", "--name", "kamal-app-multiarch")
 
-      SSHKit::Backend::Abstract.any_instance.expects(:execute)
-        .with(:docker, :buildx, :inspect, "kamal-local")
+      SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+        .with(:docker, :buildx, :inspect, "kamal-app-multiarch", "> /dev/null")
         .raises(SSHKit::Command::Failed.new("no builder"))
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute).with { |*args| args.first.start_with?("git") }
@@ -136,7 +141,7 @@ class CliBuildTest < CliTestCase
         .returns("")
 
       SSHKit::Backend::Abstract.any_instance.expects(:execute)
-        .with(:docker, :build, "--push", "--platform", "linux/amd64,linux/arm64", "--builder", "kamal-local", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".")
+        .with(:docker, :buildx, :build, "--push", "--platform", "linux/amd64,linux/arm64", "--builder", "kamal-app-multiarch", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".")
 
       run_command("push").tap do |output|
         assert_match /WARN Missing compatible builder, so creating a new one first/, output
@@ -160,7 +165,7 @@ class CliBuildTest < CliTestCase
     error = assert_raises(Kamal::Cli::HookError) { run_command("push") }
     assert_equal "Hook `pre-build` failed:\nfailed", error.message
 
-    assert @executions.none? { |args| args[0..2] == [ :docker, :build ] }
+    assert @executions.none? { |args| args[0..2] == [ :docker, :buildx, :build ] }
   end
 
   test "pull" do
@@ -202,23 +207,23 @@ class CliBuildTest < CliTestCase
 
   test "create" do
     run_command("create").tap do |output|
-      assert_match /docker buildx create --name kamal-local --driver=docker-container/, output
+      assert_match /docker buildx create --use --name kamal-app-multiarch/, output
     end
   end
 
   test "create remote" do
     run_command("create", fixture: :with_remote_builder).tap do |output|
       assert_match "Running /usr/bin/env true on 1.1.1.5", output
-      assert_match "docker context create kamal-remote-amd64-ssh---app-1-1-1-5 --description 'kamal-remote-amd64-ssh---app-1-1-1-5 host' --docker 'host=ssh://app@1.1.1.5'", output
-      assert_match "docker buildx create --name kamal-remote-amd64-ssh---app-1-1-1-5 kamal-remote-amd64-ssh---app-1-1-1-5 --platform linux/amd64", output
+      assert_match "docker context create kamal-app-native-remote-amd64 --description 'kamal-app-native-remote amd64 native host' --docker 'host=ssh://app@1.1.1.5'", output
+      assert_match "docker buildx create --name kamal-app-native-remote kamal-app-native-remote-amd64 --platform linux/amd64", output
     end
   end
 
   test "create remote with custom ports" do
     run_command("create", fixture: :with_remote_builder_and_custom_ports).tap do |output|
       assert_match "Running /usr/bin/env true on 1.1.1.5", output
-      assert_match "docker context create kamal-remote-amd64-ssh---app-1-1-1-5-2122 --description 'kamal-remote-amd64-ssh---app-1-1-1-5-2122 host' --docker 'host=ssh://app@1.1.1.5:2122'", output
-      assert_match "docker buildx create --name kamal-remote-amd64-ssh---app-1-1-1-5-2122 kamal-remote-amd64-ssh---app-1-1-1-5-2122 --platform linux/amd64", output
+      assert_match "docker context create kamal-app-native-remote-amd64 --description 'kamal-app-native-remote amd64 native host' --docker 'host=ssh://app@1.1.1.5:2122'", output
+      assert_match "docker buildx create --name kamal-app-native-remote kamal-app-native-remote-amd64 --platform linux/amd64", output
     end
   end
 
@@ -235,7 +240,7 @@ class CliBuildTest < CliTestCase
 
   test "remove" do
     run_command("remove").tap do |output|
-      assert_match /docker buildx rm kamal-local/, output
+      assert_match /docker buildx rm kamal-app-multiarch/, output
     end
   end
 
@@ -245,7 +250,7 @@ class CliBuildTest < CliTestCase
       .returns("docker builder info")
 
     run_command("details").tap do |output|
-      assert_match /Builder: local/, output
+      assert_match /Builder: multiarch/, output
       assert_match /docker builder info/, output
     end
   end

test/cli/cli_test_case.rb

@@ -36,8 +36,9 @@ class CliTestCase < ActiveSupport::TestCase
         .with { |arg1, arg2| arg1 == :mkdir && arg2 == ".kamal/locks/app" }
       SSHKit::Backend::Abstract.any_instance.stubs(:execute)
         .with { |arg1, arg2| arg1 == :rm && arg2 == ".kamal/locks/app/details" }
-      SSHKit::Backend::Abstract.any_instance.stubs(:execute)
-        .with(:docker, :buildx, :inspect, "kamal-local")
+      SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+        .with { |*args| args[0..2] == [ :docker, :buildx, :inspect ] }
+        .returns("")
     end
 
     def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: false)

test/cli/main_test.rb

@@ -121,6 +121,10 @@ class CliMainTest < CliTestCase
       .with(:git, "-C", anything, :status, "--porcelain")
       .returns("")
 
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :buildx, :inspect, "kamal-app-multiarch", "> /dev/null")
+      .returns("")
+
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :info, "--format '{{index .RegistryConfig.Mirrors 0}}'")
       .returns("")
@@ -155,6 +159,10 @@ class CliMainTest < CliTestCase
       .with(:git, "-C", anything, :status, "--porcelain")
       .returns("")
 
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :buildx, :inspect, "kamal-app-multiarch", "> /dev/null")
+      .returns("")
+
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :info, "--format '{{index .RegistryConfig.Mirrors 0}}'")
       .returns("")
@@ -482,6 +490,39 @@ class CliMainTest < CliTestCase
     end
   end
 
+  test "env files overwrite shell environment variables" do
+    ENV["TEST_VAR"] = "shell_value"
+    ENV["AWS_ACCESS_KEY_ID"] = "local_dev_key"
+
+    with_test_dotenv(".env": "TEST_VAR=dotenv_value\nAWS_ACCESS_KEY_ID=production_key") do
+      # Create a simple CLI command instance to trigger load_env
+      Kamal::Cli::Main.new.send(:load_env)
+
+      assert_equal "dotenv_value", ENV["TEST_VAR"]
+      assert_equal "production_key", ENV["AWS_ACCESS_KEY_ID"]
+    end
+  ensure
+    ENV.delete("TEST_VAR")
+    ENV.delete("AWS_ACCESS_KEY_ID")
+  end
+
+  test "destination env files overwrite base env files" do
+    ENV["TEST_VAR"] = "shell_value"
+
+    with_test_dotenv(".env": "TEST_VAR=base_value\nBASE_ONLY=base", ".env.world": "TEST_VAR=world_value\nWORLD_ONLY=world") do
+      # Create CLI command with destination to trigger load_env
+      Kamal::Cli::Main.new([], { destination: "world" }).send(:load_env)
+
+      assert_equal "world_value", ENV["TEST_VAR"]
+      assert_equal "base", ENV["BASE_ONLY"]
+      assert_equal "world", ENV["WORLD_ONLY"]
+    end
+  ensure
+    ENV.delete("TEST_VAR")
+    ENV.delete("BASE_ONLY")
+    ENV.delete("WORLD_ONLY")
+  end
+
   test "remove with confirmation" do
     run_command("remove", "-y", config_file: "deploy_with_accessories").tap do |output|
       assert_match /docker container stop traefik/, output
@@ -529,6 +570,34 @@ class CliMainTest < CliTestCase
     assert_equal Kamal::VERSION, version
   end
 
+  test "downgrade" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_with_accessories.yml", "skip_hooks" => false, "confirmed" => true, "rolling" => false }
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:downgrade", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:downgrade", [ "all" ], invoke_options)
+
+    run_command("downgrade", "-y", config_file: "deploy_with_accessories").tap do |output|
+      assert_match "Downgrading all hosts...", output
+      assert_match "Downgraded all hosts", output
+    end
+  end
+
+  test "downgrade rolling" do
+    invoke_options = { "config_file" => "test/fixtures/deploy_with_accessories.yml", "skip_hooks" => false, "confirmed" => true, "rolling" => false }
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:downgrade", [], invoke_options).times(4)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:downgrade", [ "all" ], invoke_options).times(3)
+
+    run_command("downgrade", "--rolling", "-y", config_file: "deploy_with_accessories").tap do |output|
+      assert_match "Downgrading 1.1.1.1...", output
+      assert_match "Downgraded 1.1.1.1", output
+      assert_match "Downgrading 1.1.1.2...", output
+      assert_match "Downgraded 1.1.1.2", output
+      assert_match "Downgrading 1.1.1.3...", output
+      assert_match "Downgraded 1.1.1.3", output
+      assert_match "Downgrading 1.1.1.4...", output
+      assert_match "Downgraded 1.1.1.4", output
+    end
+  end
+
   private
     def run_command(*command, config_file: "deploy_simple")
       stdouted { Kamal::Cli::Main.start([ *command, "-c", "test/fixtures/#{config_file}.yml" ]) }

test/cli/traefik_test.rb

@@ -103,6 +103,90 @@ class CliTraefikTest < CliTestCase
     end
   end
 
+  test "downgrade" do
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678")
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678")
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with { |*args| args[0..1] == [ :sh, "-c" ] }
+      .returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running").at_least_once # workers health check
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
+      .returns("") # old version
+
+    run_command("downgrade", "-y").tap do |output|
+      assert_match "Downgrading to Traefik on 1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4...", output
+      assert_match "docker login -u [REDACTED] -p [REDACTED]", output
+      assert_match "docker container stop kamal-proxy ; docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy && docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy", output
+      assert_match "docker container stop traefik", output
+      assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik", output
+      assert_match "docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik", output
+      assert_match "/usr/bin/env mkdir -p .kamal", output
+      assert_match "docker login -u [REDACTED] -p [REDACTED]", output
+      assert_match "docker container start traefik || docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" traefik:v2.11 --providers.docker --log.level=\"DEBUG\"", output
+      assert_match "/usr/bin/env mkdir -p .kamal", output
+      assert_match %r{docker rename app-web-latest app-web-latest_replaced_.*}, output
+      assert_match %r{docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-.* -e KAMAL_CONTAINER_NAME="app-web-latest" -e KAMAL_VERSION="latest" --env-file .kamal/env/roles/app-web.env --health-cmd}, output
+      assert_match "docker tag dhh/app:latest dhh/app:latest", output
+      assert_match "/usr/bin/env mkdir -p .kamal", output
+      assert_match "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done", output
+      assert_match "docker image prune --force --filter label=service=app", output
+      assert_match "Downgraded to Traefik on 1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", output
+    end
+  end
+
+  test "downgrade rolling" do
+    Object.any_instance.stubs(:sleep)
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678")
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678")
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with { |*args| args[0..1] == [ :sh, "-c" ] }
+      .returns("123") # old version
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-workers-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running").at_least_once # workers health check
+
+    SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
+      .returns("") # old version
+
+    run_command("downgrade", "--rolling", "-y",).tap do |output|
+      %w[1.1.1.1 1.1.1.2 1.1.1.3 1.1.1.4].each do |host|
+        assert_match "Downgrading to Traefik on #{host}...", output
+        assert_match "docker container stop kamal-proxy ; docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy && docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy", output
+        assert_match "Downgraded to Traefik on #{host}", output
+      end
+    end
+  end
+
   private
     def run_command(*command)
       stdouted { Kamal::Cli::Traefik.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) }

test/commands/builder_test.rb

@@ -7,49 +7,49 @@ class CommandsBuilderTest < ActiveSupport::TestCase
 
   test "target multiarch by default" do
     builder = new_builder_command(builder: { "cache" => { "type" => "gha" } })
-    assert_equal "local", builder.name
+    assert_equal "multiarch", builder.name
     assert_equal \
-      "docker build --push --platform linux/amd64,linux/arm64 --builder kamal-local -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
       builder.push.join(" ")
   end
 
   test "target native when multiarch is off" do
     builder = new_builder_command(builder: { "multiarch" => false })
-    assert_equal "local", builder.name
+    assert_equal "native", builder.name
     assert_equal \
-      "docker build --push --builder kamal-local -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .",
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
       builder.push.join(" ")
   end
 
   test "target native cached when multiarch is off and cache is set" do
     builder = new_builder_command(builder: { "multiarch" => false, "cache" => { "type" => "gha" } })
-    assert_equal "local", builder.name
+    assert_equal "native/cached", builder.name
     assert_equal \
-      "docker build --push --builder kamal-local -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
       builder.push.join(" ")
   end
 
   test "target multiarch remote when local and remote is set" do
-    builder = new_builder_command(builder: { "local" => { "arch" => "arm64" }, "remote" => { "arch" => "amd64", "host" => "ssh://app@127.0.0.1" }, "cache" => { "type" => "gha" } })
-    assert_equal "hybrid", builder.name
+    builder = new_builder_command(builder: { "local" => { "arch" => "arm64" }, "remote" => { "arch" => "amd64" }, "cache" => { "type" => "gha" } })
+    assert_equal "multiarch/remote", builder.name
     assert_equal \
-      "docker build --push --platform linux/arm64,linux/amd64 --builder kamal-hybrid-arm64-amd64-ssh---app-127-0-0-1 -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --push --platform linux/arm64,linux/amd64 --builder kamal-app-multiarch-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
       builder.push.join(" ")
   end
 
   test "target multiarch local when arch is set" do
     builder = new_builder_command(builder: { "local" => { "arch" => "amd64" } })
-    assert_equal "local", builder.name
+    assert_equal "multiarch", builder.name
     assert_equal \
-      "docker build --push --platform linux/amd64 --builder kamal-local -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --push --platform linux/amd64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .",
       builder.push.join(" ")
   end
 
   test "target native remote when only remote is set" do
-    builder = new_builder_command(builder: { "remote" => { "arch" => "amd64", "host" => "ssh://app@host" }, "cache" => { "type" => "gha" } })
-    assert_equal "remote", builder.name
+    builder = new_builder_command(builder: { "remote" => { "arch" => "amd64" }, "cache" => { "type" => "gha" } })
+    assert_equal "native/remote", builder.name
     assert_equal \
-      "docker build --push --platform linux/amd64 --builder kamal-remote-amd64-ssh---app-host -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
+      "docker buildx build --push --platform linux/amd64 --builder kamal-app-native-remote -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile .",
       builder.push.join(" ")
   end
 
@@ -93,28 +93,28 @@ class CommandsBuilderTest < ActiveSupport::TestCase
   test "build context" do
     builder = new_builder_command(builder: { "context" => ".." })
     assert_equal \
-      "docker build --push --platform linux/amd64,linux/arm64 --builder kamal-local -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ..",
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ..",
       builder.push.join(" ")
   end
 
   test "native push with build args" do
     builder = new_builder_command(builder: { "multiarch" => false, "args" => { "a" => 1, "b" => 2 } })
     assert_equal \
-      "docker build --push --builder kamal-local -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile .",
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
       builder.push.join(" ")
   end
 
   test "multiarch push with build args" do
     builder = new_builder_command(builder: { "args" => { "a" => 1, "b" => 2 } })
     assert_equal \
-      "docker build --push --platform linux/amd64,linux/arm64 --builder kamal-local -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile .",
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --build-arg a=\"1\" --build-arg b=\"2\" --file Dockerfile .",
       builder.push.join(" ")
   end
 
   test "native push with build secrets" do
     builder = new_builder_command(builder: { "multiarch" => false, "secrets" => [ "a", "b" ] })
     assert_equal \
-      "docker build --push --builder kamal-local -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile .",
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile . && docker push dhh/app:123 && docker push dhh/app:latest",
       builder.push.join(" ")
   end
 
@@ -133,31 +133,73 @@ class CommandsBuilderTest < ActiveSupport::TestCase
   test "multiarch context build" do
     builder = new_builder_command(builder: { "context" => "./foo" })
     assert_equal \
-      "docker build --push --platform linux/amd64,linux/arm64 --builder kamal-local -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo",
+      "docker buildx build --push --platform linux/amd64,linux/arm64 --builder kamal-app-multiarch -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo",
       builder.push.join(" ")
   end
 
   test "native context build" do
     builder = new_builder_command(builder: { "multiarch" => false, "context" => "./foo" })
     assert_equal \
-      "docker build --push --builder kamal-local -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo",
+      "docker build -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo && docker push dhh/app:123 && docker push dhh/app:latest",
       builder.push.join(" ")
   end
 
   test "cached context build" do
     builder = new_builder_command(builder: { "multiarch" => false, "context" => "./foo", "cache" => { "type" => "gha" } })
     assert_equal \
-      "docker build --push --builder kamal-local -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile ./foo",
+      "docker buildx build --push -t dhh/app:123 -t dhh/app:latest --cache-to type=gha --cache-from type=gha --label service=\"app\" --file Dockerfile ./foo",
       builder.push.join(" ")
   end
 
   test "remote context build" do
-    builder = new_builder_command(builder: { "remote" => { "arch" => "amd64", "host" => "ssh://app@host" }, "context" => "./foo" })
+    builder = new_builder_command(builder: { "remote" => { "arch" => "amd64" }, "context" => "./foo" })
     assert_equal \
-      "docker build --push --platform linux/amd64 --builder kamal-remote-amd64-ssh---app-host -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo",
+      "docker buildx build --push --platform linux/amd64 --builder kamal-app-native-remote -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile ./foo",
       builder.push.join(" ")
   end
 
+  test "multiarch context hosts" do
+    command = new_builder_command
+    assert_equal "docker buildx inspect kamal-app-multiarch > /dev/null", command.context_hosts.join(" ")
+    assert_equal "", command.config_context_hosts.join(" ")
+  end
+
+  test "native context hosts" do
+    command = new_builder_command(builder: { "multiarch" => false })
+    assert_equal :true, command.context_hosts
+    assert_equal "", command.config_context_hosts.join(" ")
+  end
+
+  test "native cached context hosts" do
+    command = new_builder_command(builder: { "multiarch" => false, "cache" => { "type" => "registry" } })
+    assert_equal "docker buildx inspect kamal-app-native-cached > /dev/null", command.context_hosts.join(" ")
+    assert_equal "", command.config_context_hosts.join(" ")
+  end
+
+  test "native remote context hosts" do
+    command = new_builder_command(builder: { "remote" => { "arch" => "amd64", "host" => "ssh://host" } })
+    assert_equal "docker context inspect kamal-app-native-remote-amd64 --format '{{.Endpoints.docker.Host}}'", command.context_hosts.join(" ")
+    assert_equal [ "ssh://host" ], command.config_context_hosts
+  end
+
+  test "multiarch remote context hosts" do
+    command = new_builder_command(builder: {
+      "remote" => { "arch" => "amd64", "host" => "ssh://host" },
+      "local" => { "arch" => "arm64" }
+    })
+    assert_equal "docker context inspect kamal-app-multiarch-remote-arm64 --format '{{.Endpoints.docker.Host}}' ; docker context inspect kamal-app-multiarch-remote-amd64 --format '{{.Endpoints.docker.Host}}'", command.context_hosts.join(" ")
+    assert_equal [ "ssh://host" ], command.config_context_hosts
+  end
+
+  test "multiarch remote context hosts with local host" do
+    command = new_builder_command(builder: {
+      "remote" => { "arch" => "amd64", "host" => "ssh://host" },
+      "local" => { "arch" => "arm64", "host" => "unix:///var/run/docker.sock" }
+    })
+    assert_equal "docker context inspect kamal-app-multiarch-remote-arm64 --format '{{.Endpoints.docker.Host}}' ; docker context inspect kamal-app-multiarch-remote-amd64 --format '{{.Endpoints.docker.Host}}'", command.context_hosts.join(" ")
+    assert_equal [ "unix:///var/run/docker.sock", "ssh://host" ], command.config_context_hosts
+  end
+
   test "mirror count" do
     command = new_builder_command
     assert_equal "docker info --format '{{index .RegistryConfig.Mirrors 0}}'", command.first_mirror.join(" ")

test/configuration/builder_test.rb

@@ -42,7 +42,7 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase
 
   test "setting both local and remote configs" do
     @deploy_with_builder_option[:builder] = {
-      "local" => { "arch" => "arm64" },
+      "local" => { "arch" => "arm64", "host" => "unix:///Users/<%= `whoami`.strip %>/.docker/run/docker.sock" },
       "remote" => { "arch" => "amd64", "host" => "ssh://root@192.168.0.1" }
     }
 
@@ -53,6 +53,7 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase
     assert_equal "ssh://root@192.168.0.1", config_with_builder_option.builder.remote_host
 
     assert_equal "arm64", config_with_builder_option.builder.local_arch
+    assert_equal "unix:///Users/<%= `whoami`.strip %>/.docker/run/docker.sock", config_with_builder_option.builder.local_host
   end
 
   test "cached?" do

test/integration/docker/deployer/app/config/deploy.yml

@@ -24,7 +24,6 @@ registry:
   password: root
 builder:
   multiarch: false
-  driver: docker
   args:
     COMMIT_SHA: <%= `git rev-parse HEAD` %>
 healthcheck:
@@ -34,7 +33,7 @@ traefik:
   args:
     accesslog: true
     accesslog.format: json
-  image: registry:4443/traefik:v2.10
+  image: registry:4443/traefik:v2.11
 accessories:
   busybox:
     service: custom-busybox

test/integration/docker/deployer/app_with_roles/config/deploy.yml

@@ -18,7 +18,6 @@ registry:
   password: root
 builder:
   multiarch: false
-  driver: docker
   args:
     COMMIT_SHA: <%= `git rev-parse HEAD` %>
 healthcheck:
@@ -28,7 +27,7 @@ traefik:
   args:
     accesslog: true
     accesslog.format: json
-  image: registry:4443/traefik:v2.10
+  image: registry:4443/traefik:v2.11
 accessories:
   busybox:
     service: custom-busybox

test/integration/docker/deployer/setup.sh

@@ -19,7 +19,7 @@ push_image_to_registry_4443() {
 
 install_kamal
 push_image_to_registry_4443 nginx 1-alpine-slim
-push_image_to_registry_4443 traefik v2.10
+push_image_to_registry_4443 traefik v2.11
 push_image_to_registry_4443 busybox 1.36.0
 
 # .ssh is on a shared volume that persists between runs. Clean it up as the

test/integration/docker/registry/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry
+FROM registry:3
 
 COPY boot.sh .
 

test/integration/docker/registry/boot.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-exec /entrypoint.sh /etc/docker/registry/config.yml
+exec /entrypoint.sh /etc/distribution/config.yml

test/integration/main_test.rb

@@ -32,7 +32,7 @@ class MainTest < IntegrationTest
     assert_match /Traefik Host: vm2/, details
     assert_match /App Host: vm1/, details
     assert_match /App Host: vm2/, details
-    assert_match /traefik:v2.10/, details
+    assert_match /traefik:v2.11/, details
     assert_match /registry:4443\/app:#{first_version}/, details
 
     audit = kamal :audit, capture: true

test/integration/traefik_test.rb

@@ -52,11 +52,11 @@ class TraefikTest < IntegrationTest
 
   private
     def assert_traefik_running
-      assert_match /traefik:v2.10   "\/entrypoint.sh/, traefik_details
+      assert_match /traefik:v2.11   "\/entrypoint.sh/, traefik_details
     end
 
     def assert_traefik_not_running
-      assert_no_match /traefik:v2.10   "\/entrypoint.sh/, traefik_details
+      assert_no_match /traefik:v2.11   "\/entrypoint.sh/, traefik_details
     end
 
     def traefik_details