Escape strings for the shell

Whitespaces in strings were interpreted by the shell as new words.
Escaping them ensures the shell sees the string as a single value.

.github/workflows/ci.yml

@@ -4,6 +4,7 @@ on:
     branches:
       - main
   pull_request:
+  workflow_dispatch:
 jobs:
   rubocop:
     name: RuboCop
@@ -22,35 +23,25 @@ jobs:
         run: bundle exec rubocop --parallel
   tests:
     strategy:
+      fail-fast: false
       matrix:
         ruby-version:
-          - "2.7"
-          - "3.1"
           - "3.2"
           - "3.3"
+          - "3.4"
         gemfile:
           - Gemfile
-          - gemfiles/ruby_2.7.gemfile
           - gemfiles/rails_edge.gemfile
-        exclude:
-          - ruby-version: "2.7"
-            gemfile: Gemfile
-          - ruby-version: "2.7"
-            gemfile: gemfiles/rails_edge.gemfile
-          - ruby-version: "3.1"
-            gemfile: gemfiles/ruby_2.7.gemfile
-          - ruby-version: "3.2"
-            gemfile: gemfiles/ruby_2.7.gemfile
-          - ruby-version: "3.3"
-            gemfile: gemfiles/ruby_2.7.gemfile
     name: ${{ format('Tests (Ruby {0})', matrix.ruby-version) }}
     runs-on: ubuntu-latest
-    continue-on-error: true
     env:
       BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile }}
     steps:
       - uses: actions/checkout@v4
 
+      - name: Remove gemfile.lock
+        run: rm Gemfile.lock
+
       - name: Install Ruby
         uses: ruby/setup-ruby@v1
         with:
@@ -59,3 +50,5 @@ jobs:
 
       - name: Run tests
         run: bin/test
+        env:
+          RUBYOPT: ${{ startsWith(matrix.ruby-version, '3.4.') && '--enable=frozen-string-literal' || '' }}

Dockerfile

@@ -1,7 +1,6 @@
-# Use the official Ruby 3.2.0 Alpine image as the base image
-FROM ruby:3.2.0-alpine
+FROM ruby:3.4-alpine
 
-# Install  docker/buildx-bin
+# Install docker/buildx-bin
 COPY --from=docker/buildx-bin /buildx /usr/libexec/docker/cli-plugins/docker-buildx
 
 # Set the working directory to /kamal
@@ -14,9 +13,8 @@ COPY Gemfile Gemfile.lock kamal.gemspec ./
 COPY lib/kamal/version.rb /kamal/lib/kamal/version.rb
 
 # Install system dependencies
-RUN apk add --no-cache --update build-base git docker openrc openssh-client-default \
-    && rc-update add docker boot \
-    && gem install bundler --version=2.4.3 \
+RUN apk add --no-cache build-base git docker-cli openssh-client-default yaml-dev \
+    && gem install bundler --version=2.6.5 \
     && bundle install
 
 # Copy the rest of our application code into the container.
@@ -33,7 +31,7 @@ WORKDIR /workdir
 
 # Tell git it's safe to access /workdir/.git even if
 # the directory is owned by a different user
-RUN git config --global --add safe.directory /workdir
+RUN git config --global --add safe.directory '*'
 
 # Set the entrypoint to run the installed binary in /workdir
 # Example:  docker run -it -v "$PWD:/workdir" kamal init

Gemfile.lock

@@ -1,148 +1,158 @@
 PATH
   remote: .
   specs:
-    kamal (1.4.0)
+    kamal (2.7.0)
       activesupport (>= 7.0)
       base64 (~> 0.2)
       bcrypt_pbkdf (~> 1.0)
       concurrent-ruby (~> 1.2)
-      dotenv (~> 2.8)
-      ed25519 (~> 1.2)
-      net-ssh (~> 7.0)
-      sshkit (~> 1.21)
-      thor (~> 1.2)
-      zeitwerk (~> 2.5)
+      dotenv (~> 3.1)
+      ed25519 (~> 1.4)
+      net-ssh (~> 7.3)
+      sshkit (>= 1.23.0, < 2.0)
+      thor (~> 1.3)
+      zeitwerk (>= 2.6.18, < 3.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.1.2)
-      actionview (= 7.1.2)
-      activesupport (= 7.1.2)
+    actionpack (8.0.0.1)
+      actionview (= 8.0.0.1)
+      activesupport (= 8.0.0.1)
       nokogiri (>= 1.8.5)
-      racc
       rack (>= 2.2.4)
       rack-session (>= 1.0.1)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
-    actionview (7.1.2)
-      activesupport (= 7.1.2)
+      useragent (~> 0.16)
+    actionview (8.0.0.1)
+      activesupport (= 8.0.0.1)
       builder (~> 3.1)
       erubi (~> 1.11)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
-    activesupport (7.1.2)
+    activesupport (8.0.0.1)
       base64
+      benchmark (>= 0.3)
       bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
     ast (2.4.2)
     base64 (0.2.0)
-    bcrypt_pbkdf (1.1.0)
-    bigdecimal (3.1.5)
-    builder (3.2.4)
-    concurrent-ruby (1.2.2)
+    bcrypt_pbkdf (1.1.1)
+    benchmark (0.4.0)
+    bigdecimal (3.1.8)
+    builder (3.3.0)
+    concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
     crass (1.0.6)
-    debug (1.9.1)
+    date (3.4.1)
+    debug (1.9.2)
       irb (~> 1.10)
       reline (>= 0.3.8)
-    dotenv (2.8.1)
-    drb (2.2.0)
-      ruby2_keywords
-    ed25519 (1.3.0)
-    erubi (1.12.0)
-    i18n (1.14.1)
+    dotenv (3.1.5)
+    drb (2.2.1)
+    ed25519 (1.4.0)
+    erubi (1.13.0)
+    i18n (1.14.6)
       concurrent-ruby (~> 1.0)
-    io-console (0.7.1)
-    irb (1.11.0)
-      rdoc
-      reline (>= 0.3.8)
-    json (2.7.1)
+    io-console (0.8.0)
+    irb (1.14.2)
+      rdoc (>= 4.0.0)
+      reline (>= 0.4.2)
+    json (2.9.0)
     language_server-protocol (3.17.0.3)
-    loofah (2.22.0)
+    logger (1.6.3)
+    loofah (2.23.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
-    minitest (5.20.0)
-    mocha (2.1.0)
+    minitest (5.25.4)
+    mocha (2.7.1)
       ruby2_keywords (>= 0.0.5)
-    mutex_m (0.2.0)
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
-    net-ssh (7.2.1)
-    nokogiri (1.16.0-arm64-darwin)
+    net-sftp (4.0.0)
+      net-ssh (>= 5.0.0, < 8.0.0)
+    net-ssh (7.3.0)
+    nokogiri (1.18.9-aarch64-linux-musl)
       racc (~> 1.4)
-    nokogiri (1.16.0-x86_64-darwin)
+    nokogiri (1.18.9-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.16.0-x86_64-linux)
+    nokogiri (1.18.9-x86_64-darwin)
       racc (~> 1.4)
-    parallel (1.24.0)
-    parser (3.3.0.5)
+    nokogiri (1.18.9-x86_64-linux-gnu)
+      racc (~> 1.4)
+    nokogiri (1.18.9-x86_64-linux-musl)
+      racc (~> 1.4)
+    ostruct (0.6.1)
+    parallel (1.26.3)
+    parser (3.3.6.0)
       ast (~> 2.4.1)
       racc
-    psych (5.1.2)
+    psych (5.2.1)
+      date
       stringio
-    racc (1.7.3)
-    rack (3.0.8)
-    rack-session (2.0.0)
+    racc (1.8.1)
+    rack (3.1.16)
+    rack-session (2.1.1)
+      base64 (>= 0.1.0)
       rack (>= 3.0.0)
     rack-test (2.1.0)
       rack (>= 1.3)
-    rackup (2.1.0)
+    rackup (2.2.1)
       rack (>= 3)
-      webrick (~> 1.8)
     rails-dom-testing (2.2.0)
       activesupport (>= 5.0.0)
       minitest
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.6.0)
+    rails-html-sanitizer (1.6.2)
       loofah (~> 2.21)
-      nokogiri (~> 1.14)
-    railties (7.1.2)
-      actionpack (= 7.1.2)
-      activesupport (= 7.1.2)
-      irb
+      nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
+    railties (8.0.0.1)
+      actionpack (= 8.0.0.1)
+      activesupport (= 8.0.0.1)
+      irb (~> 1.13)
       rackup (>= 1.0.0)
       rake (>= 12.2)
       thor (~> 1.0, >= 1.2.2)
       zeitwerk (~> 2.6)
     rainbow (3.1.1)
-    rake (13.1.0)
-    rdoc (6.6.2)
+    rake (13.2.1)
+    rdoc (6.8.1)
       psych (>= 4.0.0)
-    regexp_parser (2.9.0)
-    reline (0.4.2)
+    regexp_parser (2.9.3)
+    reline (0.5.12)
       io-console (~> 0.5)
-    rexml (3.2.6)
-    rubocop (1.62.1)
+    rubocop (1.69.2)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.36.2, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.2)
-      parser (>= 3.3.0.4)
-    rubocop-minitest (0.35.0)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.36.2)
+      parser (>= 3.3.1.0)
+    rubocop-minitest (0.36.0)
       rubocop (>= 1.61, < 2.0)
       rubocop-ast (>= 1.31.1, < 2.0)
-    rubocop-performance (1.20.2)
+    rubocop-performance (1.23.0)
       rubocop (>= 1.48.1, < 2.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
-    rubocop-rails (2.24.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-rails (2.27.0)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
-      rubocop (>= 1.33.0, < 2.0)
+      rubocop (>= 1.52.0, < 2.0)
       rubocop-ast (>= 1.31.1, < 2.0)
     rubocop-rails-omakase (1.0.0)
       rubocop
@@ -151,22 +161,30 @@ GEM
       rubocop-rails
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
-    sshkit (1.21.7)
-      mutex_m
+    securerandom (0.4.0)
+    sshkit (1.23.2)
+      base64
       net-scp (>= 1.1.2)
+      net-sftp (>= 2.1.2)
       net-ssh (>= 2.8.0)
-    stringio (3.1.0)
-    thor (1.3.0)
+      ostruct
+    stringio (3.1.2)
+    thor (1.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.5.0)
-    webrick (1.8.1)
-    zeitwerk (2.6.12)
+    unicode-display_width (3.1.2)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
+    uri (1.0.3)
+    useragent (0.16.11)
+    zeitwerk (2.7.1)
 
 PLATFORMS
+  aarch64-linux-musl
   arm64-darwin
   x86_64-darwin
   x86_64-linux
+  x86_64-linux-musl
 
 DEPENDENCIES
   debug
@@ -176,4 +194,4 @@ DEPENDENCIES
   rubocop-rails-omakase
 
 BUNDLED WITH
-   2.4.3
+   2.6.5

README.md

@@ -1,6 +1,6 @@
 # Kamal: Deploy web apps anywhere
 
-From bare metal to cloud VMs, deploy web apps anywhere with zero downtime. Kamal has the dynamic reverse-proxy Traefik hold requests while a new app container is started and the old one is stopped. Works seamlessly across multiple hosts, using SSHKit to execute commands. Originally built for Rails apps, Kamal will work with any type of web app that can be containerized with Docker.
+From bare metal to cloud VMs, deploy web apps anywhere with zero downtime. Kamal uses [kamal-proxy](https://github.com/basecamp/kamal-proxy) to seamlessly switch requests between containers. Works seamlessly across multiple servers, using SSHKit to execute commands. Originally built for Rails apps, Kamal will work with any type of web app that can be containerized with Docker.
 
 ➡️ See [kamal-deploy.org](https://kamal-deploy.org) for documentation on [installation](https://kamal-deploy.org/docs/installation), [configuration](https://kamal-deploy.org/docs/configuration), and [commands](https://kamal-deploy.org/docs/commands).
 

bin/docs

@@ -0,0 +1,141 @@
+#!/usr/bin/env ruby
+require "stringio"
+
+def usage
+  puts "Usage: #{$0} <kamal_site_repo>"
+  exit 1
+end
+
+usage if ARGV.size != 1
+
+kamal_site_repo = ARGV[0]
+
+if !File.directory?(kamal_site_repo)
+  puts "Error: #{kamal_site_repo} is not a directory"
+  exit 1
+end
+
+DOCS = {
+  "accessory" => "Accessories",
+  "alias" => "Aliases",
+  "boot" => "Booting",
+  "builder" => "Builders",
+  "configuration" => "Configuration overview",
+  "env" => "Environment variables",
+  "logging" => "Logging",
+  "proxy" => "Proxy",
+  "registry" => "Docker Registry",
+  "role" => "Roles",
+  "servers" => "Servers",
+  "ssh" => "SSH",
+  "sshkit" => "SSHKit"
+}
+DOCS_PATH = "lib/kamal/configuration/docs"
+
+class DocWriter
+  attr_reader :from_file, :to_file, :key, :heading, :body, :output, :in_yaml
+
+  def initialize(from_file, to_dir)
+    @from_file = from_file
+    @key = File.basename(from_file, ".yml")
+    @to_file = File.join(to_dir, "#{linkify(DOCS[key])}.md")
+    @body = File.readlines(from_file)
+    @heading = body.shift.chomp("\n")
+    @output = nil
+  end
+
+  def write
+    puts "Writing #{to_file}"
+    generate_markdown
+    File.write(to_file, output.string)
+  end
+
+  private
+    def generate_markdown
+      @output = StringIO.new
+
+      generate_header
+
+      place = :in_section
+
+      loop do
+        line = body.shift&.chomp("\n")
+        break if line.nil?
+
+        case place
+        when :new_section, :in_section
+          if line.empty?
+            output.puts
+            place = :new_section
+          elsif line =~ /^ *#/
+            generate_line(line, heading: place == :new_section)
+            place = :in_section
+          else
+            output.puts
+            output.puts "```yaml"
+            output.puts line
+            place = :in_yaml
+          end
+        when :in_yaml, :in_empty_line_yaml
+          if line =~ /^ *#/
+            output.puts "```"
+            output.puts
+            generate_line(line, heading: place == :in_empty_line_yaml)
+            place = :in_section
+          elsif line.empty?
+            place = :in_empty_line_yaml
+          else
+            output.puts line
+          end
+        end
+      end
+
+      output.puts "```" if place == :in_yaml
+    end
+
+    def generate_header
+      output.puts "---"
+      output.puts "# This file has been generated from the Kamal source, do not edit directly."
+      output.puts "# Find the source of this file at #{DOCS_PATH}/#{key}.yml in the Kamal repository."
+      output.puts "title: #{heading[2..-1]}"
+      output.puts "---"
+      output.puts
+      output.puts heading
+    end
+
+    def generate_line(line, heading: false)
+      line = line.gsub(/^ *#\s?/, "")
+
+      if line =~ /(.*)kamal docs ([a-z]*)(.*)/
+        line = "#{$1}[#{DOCS[$2]}](../#{linkify(DOCS[$2])})#{$3}"
+      end
+
+      if line =~ /(.*)https:\/\/kamal-deploy.org([a-z\/-]*)(.*)/
+        line = "#{$1}[#{titlify($2.split("/").last)}](#{$2})#{$3}"
+      end
+
+      if heading
+        output.puts "## [#{line}](##{linkify(line)})"
+      else
+        output.puts line
+      end
+    end
+
+    def linkify(text)
+      if text == "Configuration overview"
+        "overview"
+      else
+        text.downcase.gsub(" ", "-")
+      end
+    end
+
+    def titlify(text)
+      text.capitalize.gsub("-", " ")
+    end
+end
+
+from_dir = File.join(File.dirname(__FILE__), "../#{DOCS_PATH}")
+to_dir = File.join(kamal_site_repo, "docs/configuration")
+Dir.glob("#{from_dir}/*") do |from_file|
+  DocWriter.new(from_file, to_dir).write
+end

gemfiles/ruby_2.7.gemfile

@@ -1,6 +0,0 @@
-source 'https://rubygems.org'
-git_source(:github) { |repo| "https://github.com/#{repo}.git" }
-
-gemspec path: "../"
-
-gem "nokogiri", "~> 1.15.0"

kamal.gemspec

@@ -12,12 +12,12 @@ Gem::Specification.new do |spec|
   spec.executables = %w[ kamal ]
 
   spec.add_dependency "activesupport", ">= 7.0"
-  spec.add_dependency "sshkit", "~> 1.21"
-  spec.add_dependency "net-ssh", "~> 7.0"
-  spec.add_dependency "thor", "~> 1.2"
-  spec.add_dependency "dotenv", "~> 2.8"
-  spec.add_dependency "zeitwerk", "~> 2.5"
-  spec.add_dependency "ed25519", "~> 1.2"
+  spec.add_dependency "sshkit", ">= 1.23.0", "< 2.0"
+  spec.add_dependency "net-ssh", "~> 7.3"
+  spec.add_dependency "thor", "~> 1.3"
+  spec.add_dependency "dotenv", "~> 3.1"
+  spec.add_dependency "zeitwerk", ">= 2.6.18", "< 3.0"
+  spec.add_dependency "ed25519", "~> 1.4"
   spec.add_dependency "bcrypt_pbkdf", "~> 1.0"
   spec.add_dependency "concurrent-ruby", "~> 1.2"
   spec.add_dependency "base64", "~> 0.2"

lib/kamal.rb

@@ -1,10 +1,14 @@
 module Kamal
+  class ConfigurationError < StandardError; end
 end
 
 require "active_support"
 require "zeitwerk"
+require "yaml"
+require "tmpdir"
+require "pathname"
 
 loader = Zeitwerk::Loader.for_gem
-loader.ignore("#{__dir__}/kamal/sshkit_with_ext.rb")
+loader.ignore(File.join(__dir__, "kamal", "sshkit_with_ext.rb"))
 loader.setup
-loader.eager_load # We need all commands loaded.
+loader.eager_load_namespace(Kamal::Cli) # We need all commands loaded.

lib/kamal/cli.rb

@@ -1,6 +1,8 @@
 module Kamal::Cli
-  class LockError < StandardError; end
+  class BootError < StandardError; end
   class HookError < StandardError; end
+  class LockError < StandardError; end
+  class DependencyError < StandardError; end
 end
 
 # SSHKit uses instance eval, so we need a global const for ergonomics

lib/kamal/cli/accessory.rb

@@ -1,18 +1,39 @@
+require "active_support/core_ext/array/conversions"
+require "concurrent/array"
+
 class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "boot [NAME]", "Boot new accessory service on host (use NAME=all to boot all accessories)"
-  def boot(name, login: true)
-    mutating do
+  def boot(name, prepare: true)
+    with_lock do
       if name == "all"
         KAMAL.accessory_names.each { |accessory_name| boot(accessory_name) }
       else
+        prepare(name) if prepare
+
         with_accessory(name) do |accessory, hosts|
+          booted_hosts = Concurrent::Array.new
+          on(hosts) do |host|
+            booted_hosts << host.to_s if capture_with_info(*accessory.info(all: true, quiet: true)).strip.presence
+          end
+
+          if booted_hosts.any?
+            say "Skipping booting `#{name}` on #{booted_hosts.sort.join(", ")}, a container already exists", :yellow
+            hosts -= booted_hosts
+          end
+
           directories(name)
           upload(name)
 
-          on(hosts) do
-            execute *KAMAL.registry.login if login
+          on(hosts) do |host|
             execute *KAMAL.auditor.record("Booted #{name} accessory"), verbosity: :debug
-            execute *accessory.run
+            execute *accessory.ensure_env_directory
+            upload! accessory.secrets_io, accessory.secrets_path, mode: "0600"
+            execute *accessory.run(host: host)
+
+            if accessory.running_proxy?
+              target = capture_with_info(*accessory.container_id_for(container_name: accessory.service_name, only_running: true)).strip
+              execute *accessory.deploy(target: target)
+            end
           end
         end
       end
@@ -21,7 +42,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "upload [NAME]", "Upload accessory files to host", hide: true
   def upload(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           accessory.files.each do |(local, remote)|
@@ -38,7 +59,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "directories [NAME]", "Create accessory directories on host", hide: true
   def directories(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           accessory.directories.keys.each do |host_path|
@@ -51,30 +72,29 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container; use NAME=all to boot all accessories)"
   def reboot(name)
-    mutating do
+    with_lock do
       if name == "all"
         KAMAL.accessory_names.each { |accessory_name| reboot(accessory_name) }
       else
-        with_accessory(name) do |accessory, hosts|
-          on(hosts) do
-            execute *KAMAL.registry.login
-          end
-
-          stop(name)
-          remove_container(name)
-          boot(name, login: false)
-        end
+        prepare(name)
+        stop(name)
+        remove_container(name)
+        boot(name, prepare: false)
       end
     end
   end
 
   desc "start [NAME]", "Start existing accessory container on host"
   def start(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *KAMAL.auditor.record("Started #{name} accessory"), verbosity: :debug
           execute *accessory.start
+          if accessory.running_proxy?
+            target = capture_with_info(*accessory.container_id_for(container_name: accessory.service_name, only_running: true)).strip
+            execute *accessory.deploy(target: target)
+          end
         end
       end
     end
@@ -82,11 +102,16 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "stop [NAME]", "Stop existing accessory container on host"
   def stop(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *KAMAL.auditor.record("Stopped #{name} accessory"), verbosity: :debug
           execute *accessory.stop, raise_on_non_zero_exit: false
+
+          if accessory.running_proxy?
+            target = capture_with_info(*accessory.container_id_for(container_name: accessory.service_name, only_running: true)).strip
+            execute *accessory.remove if target
+          end
         end
       end
     end
@@ -94,11 +119,9 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "restart [NAME]", "Restart existing accessory container on host"
   def restart(name)
-    mutating do
-      with_accessory(name) do
-        stop(name)
-        start(name)
-      end
+    with_lock do
+      stop(name)
+      start(name)
     end
   end
 
@@ -107,38 +130,44 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     if name == "all"
       KAMAL.accessory_names.each { |accessory_name| details(accessory_name) }
     else
+      type = "Accessory #{name}"
       with_accessory(name) do |accessory, hosts|
-        on(hosts) { puts capture_with_info(*accessory.info) }
+        on(hosts) { puts_by_host host, capture_with_info(*accessory.info), type: type }
       end
     end
   end
 
-  desc "exec [NAME] [CMD]", "Execute a custom command on servers (use --help to show options)"
+  desc "exec [NAME] [CMD...]", "Execute a custom command on servers within the accessory container (use --help to show options)"
   option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
   option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
-  def exec(name, cmd)
+  def exec(name, *cmd)
+    pre_connect_if_required
+
+    cmd = Kamal::Utils.join_commands(cmd)
     with_accessory(name) do |accessory, hosts|
       case
       when options[:interactive] && options[:reuse]
-        say "Launching interactive command with via SSH from existing container...", :magenta
+        say "Launching interactive command via SSH from existing container...", :magenta
         run_locally { exec accessory.execute_in_existing_container_over_ssh(cmd) }
 
       when options[:interactive]
         say "Launching interactive command via SSH from new container...", :magenta
+        on(accessory.hosts.first) { execute *KAMAL.registry.login }
         run_locally { exec accessory.execute_in_new_container_over_ssh(cmd) }
 
       when options[:reuse]
         say "Launching command from existing container...", :magenta
-        on(hosts) do
+        on(hosts) do |host|
           execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
-          capture_with_info(*accessory.execute_in_existing_container(cmd))
+          puts_by_host host, capture_with_info(*accessory.execute_in_existing_container(cmd))
         end
 
       else
         say "Launching command from new container...", :magenta
-        on(hosts) do
+        on(hosts) do |host|
+          execute *KAMAL.registry.login
           execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
-          capture_with_info(*accessory.execute_in_new_container(cmd))
+          puts_by_host host, capture_with_info(*accessory.execute_in_new_container(cmd))
         end
       end
     end
@@ -148,23 +177,27 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
   option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server"
   option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
+  option :grep_options, desc: "Additional options supplied to grep"
   option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
+  option :skip_timestamps, type: :boolean, aliases: "-T", desc: "Skip appending timestamps to logging output"
   def logs(name)
     with_accessory(name) do |accessory, hosts|
       grep = options[:grep]
+      grep_options = options[:grep_options]
+      timestamps = !options[:skip_timestamps]
 
       if options[:follow]
         run_locally do
           info "Following logs on #{hosts}..."
-          info accessory.follow_logs(grep: grep)
-          exec accessory.follow_logs(grep: grep)
+          info accessory.follow_logs(timestamps: timestamps, grep: grep, grep_options: grep_options)
+          exec accessory.follow_logs(timestamps: timestamps, grep: grep, grep_options: grep_options)
         end
       else
         since = options[:since]
         lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
         on(hosts) do
-          puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep))
+          puts capture_with_info(*accessory.logs(timestamps: timestamps, since: since, lines: lines, grep: grep, grep_options: grep_options))
         end
       end
     end
@@ -173,17 +206,12 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "remove [NAME]", "Remove accessory container, image and data directory from host (use NAME=all to remove all accessories)"
   option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def remove(name)
-    mutating do
-      if name == "all"
-        KAMAL.accessory_names.each { |accessory_name| remove(accessory_name) }
-      else
-        confirming "This will remove all containers, images and data directories for #{name}. Are you sure?" do
-          with_accessory(name) do
-            stop(name)
-            remove_container(name)
-            remove_image(name)
-            remove_service_directory(name)
-          end
+    confirming "This will remove all containers, images and data directories for #{name}. Are you sure?" do
+      with_lock do
+        if name == "all"
+          KAMAL.accessory_names.each { |accessory_name| remove_accessory(accessory_name) }
+        else
+          remove_accessory(name)
         end
       end
     end
@@ -191,7 +219,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "remove_container [NAME]", "Remove accessory container from host", hide: true
   def remove_container(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *KAMAL.auditor.record("Remove #{name} accessory container"), verbosity: :debug
@@ -203,7 +231,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "remove_image [NAME]", "Remove accessory image from host", hide: true
   def remove_image(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *KAMAL.auditor.record("Removed #{name} accessory image"), verbosity: :debug
@@ -215,7 +243,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "remove_service_directory [NAME]", "Remove accessory directory used for uploaded files and data directories from host", hide: true
   def remove_service_directory(name)
-    mutating do
+    with_lock do
       with_accessory(name) do |accessory, hosts|
         on(hosts) do
           execute *accessory.remove_service_directory
@@ -224,6 +252,25 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     end
   end
 
+  desc "upgrade", "Upgrade accessories from Kamal 1.x to 2.0 (restart them in 'kamal' network)"
+  option :rolling, type: :boolean, default: false, desc: "Upgrade one host at a time"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  def upgrade(name)
+    confirming "This will restart all accessories" do
+      with_lock do
+        host_groups = options[:rolling] ? KAMAL.accessory_hosts : [ KAMAL.accessory_hosts ]
+        host_groups.each do |hosts|
+          host_list = Array(hosts).join(",")
+          KAMAL.with_specific_hosts(hosts) do
+            say "Upgrading #{name} accessories on #{host_list}...", :magenta
+            reboot name
+            say "Upgraded #{name} accessories on #{host_list}...", :magenta
+          end
+        end
+      end
+    end
+  end
+
   private
     def with_accessory(name)
       if KAMAL.config.accessory(name)
@@ -243,10 +290,24 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     end
 
     def accessory_hosts(accessory)
-      if KAMAL.specific_hosts&.any?
-        KAMAL.specific_hosts & accessory.hosts
-      else
-        accessory.hosts
+      KAMAL.accessory_hosts & accessory.hosts
+    end
+
+    def remove_accessory(name)
+      stop(name)
+      remove_container(name)
+      remove_image(name)
+      remove_service_directory(name)
+    end
+
+    def prepare(name)
+      with_accessory(name) do |accessory, hosts|
+        on(hosts) do
+          execute *KAMAL.registry.login(registry_config: accessory.registry)
+          execute *KAMAL.docker.create_network
+        rescue SSHKit::Command::Failed => e
+          raise unless e.message.include?("already exists")
+        end
       end
     end
 end

lib/kamal/cli/alias/command.rb

@@ -0,0 +1,10 @@
+class Kamal::Cli::Alias::Command < Thor::DynamicCommand
+  def run(instance, args = [])
+    if (_alias = KAMAL.config.aliases[name])
+      KAMAL.reset
+      Kamal::Cli::Main.start(Shellwords.split(_alias.command) + ARGV[1..-1])
+    else
+      super
+    end
+  end
+end

lib/kamal/cli/app.rb

@@ -1,27 +1,42 @@
 class Kamal::Cli::App < Kamal::Cli::Base
   desc "boot", "Boot app on servers (or reboot app if already running)"
   def boot
-    mutating do
-      hold_lock_on_error do
-        say "Get most recent version available as an image...", :magenta unless options[:version]
-        using_version(version_or_latest) do |version|
-          say "Start container with version #{version} using a #{KAMAL.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
+    with_lock do
+      say "Get most recent version available as an image...", :magenta unless options[:version]
+      using_version(version_or_latest) do |version|
+        say "Start container with version #{version} (or reboot if already running)...", :magenta
 
-          # Assets are prepared in a separate step to ensure they are on all hosts before booting
-          on(KAMAL.hosts) do
-            execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
-            execute *KAMAL.app.tag_current_image_as_latest
+        # Assets are prepared in a separate step to ensure they are on all hosts before booting
+        on(KAMAL.app_hosts) do
+          Kamal::Cli::App::ErrorPages.new(host, self).run
 
+          KAMAL.roles_on(host).each do |role|
+            Kamal::Cli::App::Assets.new(host, role, self).run
+            Kamal::Cli::App::SslCertificates.new(host, role, self).run
+          end
+        end
+
+        # Primary hosts and roles are returned first, so they can open the barrier
+        barrier = Kamal::Cli::Healthcheck::Barrier.new
+
+        host_boot_groups.each do |hosts|
+          host_list = Array(hosts).join(",")
+          run_hook "pre-app-boot", hosts: host_list
+
+          on(hosts) do |host|
             KAMAL.roles_on(host).each do |role|
-              Kamal::Cli::App::PrepareAssets.new(host, role, self).run
+              Kamal::Cli::App::Boot.new(host, role, self, version, barrier).run
             end
           end
 
-          on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
-            KAMAL.roles_on(host).each do |role|
-              Kamal::Cli::App::Boot.new(host, role, version, self).run
-            end
-          end
+          run_hook "post-app-boot", hosts: host_list
+          sleep KAMAL.config.boot.wait if KAMAL.config.boot.wait
+        end
+
+        # Tag once the app booted on all hosts
+        on(KAMAL.app_hosts) do |host|
+          execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
+          execute *KAMAL.app.tag_latest_image
         end
       end
     end
@@ -29,13 +44,22 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "start", "Start existing app container on servers"
   def start
-    mutating do
-      on(KAMAL.hosts) do |host|
+    with_lock do
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
+          app = KAMAL.app(role: role, host: host)
           execute *KAMAL.auditor.record("Started app version #{KAMAL.config.version}"), verbosity: :debug
-          execute *KAMAL.app(role: role).start, raise_on_non_zero_exit: false
+          execute *app.start, raise_on_non_zero_exit: false
+
+          if role.running_proxy?
+            version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+            endpoint = capture_with_info(*app.container_id_for_version(version)).strip
+            raise Kamal::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
+
+            execute *app.deploy(target: endpoint)
+          end
         end
       end
     end
@@ -43,13 +67,23 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "stop", "Stop app container on servers"
   def stop
-    mutating do
-      on(KAMAL.hosts) do |host|
+    with_lock do
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
+          app = KAMAL.app(role: role, host: host)
           execute *KAMAL.auditor.record("Stopped app", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).stop, raise_on_non_zero_exit: false
+
+          if role.running_proxy?
+            version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+            endpoint = capture_with_info(*app.container_id_for_version(version)).strip
+            if endpoint.present?
+              execute *app.remove, raise_on_non_zero_exit: false
+            end
+          end
+
+          execute *app.stop, raise_on_non_zero_exit: false
         end
       end
     end
@@ -58,33 +92,49 @@ class Kamal::Cli::App < Kamal::Cli::Base
   # FIXME: Drop in favor of just containers?
   desc "details", "Show details about app containers"
   def details
-    on(KAMAL.hosts) do |host|
+    on(KAMAL.app_hosts) do |host|
       roles = KAMAL.roles_on(host)
 
       roles.each do |role|
-        puts_by_host host, capture_with_info(*KAMAL.app(role: role).info)
+        puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).info)
       end
     end
   end
 
-  desc "exec [CMD]", "Execute a custom command on servers (use --help to show options)"
+  desc "exec [CMD...]", "Execute a custom command on servers within the app container (use --help to show options)"
   option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
   option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
-  def exec(cmd)
+  option :env, aliases: "-e", type: :hash, desc: "Set environment variables for the command"
+  option :detach, type: :boolean, default: false, desc: "Execute command in a detached container"
+  def exec(*cmd)
+    pre_connect_if_required
+
+    if (incompatible_options = [ :interactive, :reuse ].select { |key| options[:detach] && options[key] }.presence)
+      raise ArgumentError, "Detach is not compatible with #{incompatible_options.join(" or ")}"
+    end
+
+    if cmd.empty?
+      raise ArgumentError, "No command provided. You must specify a command to execute."
+    end
+
+    cmd = Kamal::Utils.join_commands(cmd)
+    env = options[:env]
+    detach = options[:detach]
     case
     when options[:interactive] && options[:reuse]
       say "Get current version of running container...", :magenta unless options[:version]
       using_version(options[:version] || current_running_version) do |version|
         say "Launching interactive command with version #{version} via SSH from existing container on #{KAMAL.primary_host}...", :magenta
-        run_locally { exec KAMAL.app(role: KAMAL.primary_role).execute_in_existing_container_over_ssh(cmd, host: KAMAL.primary_host) }
+        run_locally { exec KAMAL.app(role: KAMAL.primary_role, host: KAMAL.primary_host).execute_in_existing_container_over_ssh(cmd, env: env) }
       end
 
     when options[:interactive]
       say "Get most recent version available as an image...", :magenta unless options[:version]
       using_version(version_or_latest) do |version|
         say "Launching interactive command with version #{version} via SSH from new container on #{KAMAL.primary_host}...", :magenta
+        on(KAMAL.primary_host) { execute *KAMAL.registry.login }
         run_locally do
-          exec KAMAL.app(role: KAMAL.primary_role).execute_in_new_container_over_ssh(cmd, host: KAMAL.primary_host)
+          exec KAMAL.app(role: KAMAL.primary_role, host: KAMAL.primary_host).execute_in_new_container_over_ssh(cmd, env: env)
         end
       end
 
@@ -93,12 +143,12 @@ class Kamal::Cli::App < Kamal::Cli::Base
       using_version(options[:version] || current_running_version) do |version|
         say "Launching command with version #{version} from existing container...", :magenta
 
-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
           roles = KAMAL.roles_on(host)
 
           roles.each do |role|
             execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}", role: role), verbosity: :debug
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_existing_container(cmd))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).execute_in_existing_container(cmd, env: env))
           end
         end
       end
@@ -107,12 +157,14 @@ class Kamal::Cli::App < Kamal::Cli::Base
       say "Get most recent version available as an image...", :magenta unless options[:version]
       using_version(version_or_latest) do |version|
         say "Launching command with version #{version} from new container...", :magenta
-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
+          execute *KAMAL.registry.login
+
           roles = KAMAL.roles_on(host)
 
           roles.each do |role|
             execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_new_container(cmd))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).execute_in_new_container(cmd, env: env, detach: detach))
           end
         end
       end
@@ -121,25 +173,27 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "containers", "Show app containers on servers"
   def containers
-    on(KAMAL.hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_containers) }
+    on(KAMAL.app_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_containers) }
   end
 
   desc "stale_containers", "Detect app stale containers"
   option :stop, aliases: "-s", type: :boolean, default: false, desc: "Stop the stale containers found"
   def stale_containers
-    mutating do
-      stop = options[:stop]
+    stop = options[:stop]
 
-      cli = self
-
-      on(KAMAL.hosts) do |host|
+    with_lock_if_stopping do
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
-          cli.send(:stale_versions, host: host, role: role).each do |version|
+          app = KAMAL.app(role: role, host: host)
+          versions = capture_with_info(*app.list_versions, raise_on_non_zero_exit: false).split("\n")
+          versions -= [ capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip ]
+
+          versions.each do |version|
             if stop
               puts_by_host host, "Stopping stale container for role #{role} with version #{version}"
-              execute *KAMAL.app(role: role).stop(version: version), raise_on_non_zero_exit: false
+              execute *app.stop(version: version), raise_on_non_zero_exit: false
             else
               puts_by_host host,  "Detected stale container for role #{role} with version #{version} (use `kamal app stale_containers --stop` to stop)"
             end
@@ -151,40 +205,48 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "images", "Show app images on servers"
   def images
-    on(KAMAL.hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_images) }
+    on(KAMAL.app_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.app.list_images) }
   end
 
   desc "logs", "Show log lines from app on servers (use --help to show options)"
   option :since, aliases: "-s", desc: "Show lines since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
   option :lines, type: :numeric, aliases: "-n", desc: "Number of lines to show from each server"
   option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
+  option :grep_options, desc: "Additional options supplied to grep"
   option :follow, aliases: "-f", desc: "Follow log on primary server (or specific host set by --hosts)"
+  option :skip_timestamps, type: :boolean, aliases: "-T", desc: "Skip appending timestamps to logging output"
+  option :container_id, desc: "Docker container ID to fetch logs"
   def logs
     # FIXME: Catch when app containers aren't running
 
     grep = options[:grep]
+    grep_options = options[:grep_options]
     since = options[:since]
+    container_id = options[:container_id]
+    timestamps = !options[:skip_timestamps]
+
     if options[:follow]
       lines = options[:lines].presence || ((since || grep) ? nil : 10) # Default to 10 lines if since or grep isn't set
 
       run_locally do
         info "Following logs on #{KAMAL.primary_host}..."
 
-        KAMAL.specific_roles ||= [ "web" ]
+        KAMAL.specific_roles ||= [ KAMAL.primary_role.name ]
         role = KAMAL.roles_on(KAMAL.primary_host).first
 
-        info KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
-        exec KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep)
+        app = KAMAL.app(role: role, host: host)
+        info app.follow_logs(host: KAMAL.primary_host, container_id: container_id, timestamps: timestamps, lines: lines, grep: grep, grep_options: grep_options)
+        exec app.follow_logs(host: KAMAL.primary_host, container_id: container_id, timestamps: timestamps, lines: lines, grep: grep, grep_options: grep_options)
       end
     else
       lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
-      on(KAMAL.hosts) do |host|
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
           begin
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).logs(since: since, lines: lines, grep: grep))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).logs(container_id: container_id, timestamps: timestamps, since: since, lines: lines, grep: grep, grep_options: grep_options))
           rescue SSHKit::Command::Failed
             puts_by_host host, "Nothing found"
           end
@@ -195,22 +257,53 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove", "Remove app containers and images from servers"
   def remove
-    mutating do
+    with_lock do
       stop
       remove_containers
       remove_images
+      remove_app_directories
+    end
+  end
+
+  desc "live", "Set the app to live mode"
+  def live
+    with_lock do
+      on(KAMAL.proxy_hosts) do |host|
+        roles = KAMAL.roles_on(host)
+
+        roles.each do |role|
+          execute *KAMAL.app(role: role, host: host).live if role.running_proxy?
+        end
+      end
+    end
+  end
+
+  desc "maintenance", "Set the app to maintenance mode"
+  option :drain_timeout, type: :numeric, desc: "How long to allow in-flight requests to complete (defaults to drain_timeout from config)"
+  option :message, type: :string, desc: "Message to display to clients while stopped"
+  def maintenance
+    maintenance_options = { drain_timeout: options[:drain_timeout] || KAMAL.config.drain_timeout, message: options[:message] }
+
+    with_lock do
+      on(KAMAL.proxy_hosts) do |host|
+        roles = KAMAL.roles_on(host)
+
+        roles.each do |role|
+          execute *KAMAL.app(role: role, host: host).maintenance(**maintenance_options) if role.running_proxy?
+        end
+      end
     end
   end
 
   desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true
   def remove_container(version)
-    mutating do
-      on(KAMAL.hosts) do |host|
+    with_lock do
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
           execute *KAMAL.auditor.record("Removed app container with version #{version}", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).remove_container(version: version)
+          execute *KAMAL.app(role: role, host: host).remove_container(version: version)
         end
       end
     end
@@ -218,13 +311,13 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove_containers", "Remove all app containers from servers", hide: true
   def remove_containers
-    mutating do
-      on(KAMAL.hosts) do |host|
+    with_lock do
+      on(KAMAL.app_hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
           execute *KAMAL.auditor.record("Removed all app containers", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).remove_containers
+          execute *KAMAL.app(role: role, host: host).remove_containers
         end
       end
     end
@@ -232,19 +325,36 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove_images", "Remove all app images from servers", hide: true
   def remove_images
-    mutating do
-      on(KAMAL.hosts) do
+    with_lock do
+      on(KAMAL.app_hosts) do
         execute *KAMAL.auditor.record("Removed all app images"), verbosity: :debug
         execute *KAMAL.app.remove_images
       end
     end
   end
 
+  desc "remove_app_directories", "Remove the app directories from servers", hide: true
+  def remove_app_directories
+    with_lock do
+      on(KAMAL.app_hosts) do |host|
+        roles = KAMAL.roles_on(host)
+
+        roles.each do |role|
+          execute *KAMAL.auditor.record("Removed #{KAMAL.config.app_directory}", role: role), verbosity: :debug
+          execute *KAMAL.server.remove_app_directory, raise_on_non_zero_exit: false
+        end
+
+        execute *KAMAL.auditor.record("Removed #{KAMAL.config.app_directory}"), verbosity: :debug
+        execute *KAMAL.app.remove_proxy_app_directory, raise_on_non_zero_exit: false
+      end
+    end
+  end
+
   desc "version", "Show app version currently running on servers"
   def version
-    on(KAMAL.hosts) do |host|
+    on(KAMAL.app_hosts) do |host|
       role = KAMAL.roles_on(host).first
-      puts_by_host host, capture_with_info(*KAMAL.app(role: role).current_running_version).strip
+      puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).current_running_version).strip
     end
   end
 
@@ -267,23 +377,24 @@ class Kamal::Cli::App < Kamal::Cli::Base
       version = nil
       on(host) do
         role = KAMAL.roles_on(host).first
-        version = capture_with_info(*KAMAL.app(role: role).current_running_version).strip
+        version = capture_with_info(*KAMAL.app(role: role, host: host).current_running_version).strip
       end
       version.presence
     end
 
-    def stale_versions(host:, role:)
-      versions = nil
-      on(host) do
-        versions = \
-          capture_with_info(*KAMAL.app(role: role).list_versions, raise_on_non_zero_exit: false)
-          .split("\n")
-          .drop(1)
-      end
-      versions
+    def version_or_latest
+      options[:version] || KAMAL.config.latest_tag
     end
 
-    def version_or_latest
-      options[:version] || "latest"
+    def with_lock_if_stopping
+      if options[:stop]
+        with_lock { yield }
+      else
+        yield
+      end
+    end
+
+    def host_boot_groups
+      KAMAL.config.boot.limit ? KAMAL.app_hosts.each_slice(KAMAL.config.boot.limit).to_a : [ KAMAL.app_hosts ]
     end
 end

lib/kamal/cli/app/assets.rb

@@ -1,4 +1,4 @@
-class Kamal::Cli::App::PrepareAssets
+class Kamal::Cli::App::Assets
   attr_reader :host, :role, :sshkit
   delegate :execute, :capture_with_info, :info, to: :sshkit
   delegate :assets?, to: :role
@@ -19,6 +19,6 @@ class Kamal::Cli::App::PrepareAssets
 
   private
     def app
-      @app ||= KAMAL.app(role: role)
+      @app ||= KAMAL.app(role: role, host: host)
     end
 end

lib/kamal/cli/app/boot.rb

@@ -1,19 +1,30 @@
 class Kamal::Cli::App::Boot
-  attr_reader :host, :role, :version, :sshkit
-  delegate :execute, :capture_with_info, :info, to: :sshkit
-  delegate :uses_cord?, :assets?, to: :role
+  attr_reader :host, :role, :version, :barrier, :sshkit
+  delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, :upload!, to: :sshkit
+  delegate :assets?, :running_proxy?, to: :role
 
-  def initialize(host, role, version, sshkit)
+  def initialize(host, role, sshkit, version, barrier)
     @host = host
     @role = role
     @version = version
+    @barrier = barrier
     @sshkit = sshkit
   end
 
   def run
     old_version = old_version_renamed_if_clashing
 
-    start_new_version
+    wait_at_barrier if queuer?
+
+    begin
+      start_new_version
+    rescue => e
+      close_barrier if gatekeeper?
+      stop_new_version
+      raise
+    end
+
+    release_barrier if gatekeeper?
 
     if old_version
       stop_old_version(old_version)
@@ -21,18 +32,6 @@ class Kamal::Cli::App::Boot
   end
 
   private
-    def app
-      @app ||= KAMAL.app(role: role)
-    end
-
-    def auditor
-      @auditor = KAMAL.auditor(role: role)
-    end
-
-    def audit(message)
-      execute *auditor.record(message), verbosity: :debug
-    end
-
     def old_version_renamed_if_clashing
       if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
         renamed_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
@@ -46,22 +45,82 @@ class Kamal::Cli::App::Boot
 
     def start_new_version
       audit "Booted app version #{version}"
-      execute *app.tie_cord(role.cord_host_file) if uses_cord?
-      execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
-      Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+      hostname = "#{host.to_s[0...51].chomp(".")}-#{SecureRandom.hex(6)}"
+
+      execute *app.ensure_env_directory
+      upload! role.secrets_io(host), role.secrets_path, mode: "0600"
+
+      execute *app.run(hostname: hostname)
+      if running_proxy?
+        endpoint = capture_with_info(*app.container_id_for_version(version)).strip
+        raise Kamal::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
+        execute *app.deploy(target: endpoint)
+      else
+        Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+      end
+    rescue => e
+      error "Failed to boot #{role} on #{host}"
+      raise e
+    end
+
+    def stop_new_version
+      execute *app.stop(version: version), raise_on_non_zero_exit: false
     end
 
     def stop_old_version(version)
-      if uses_cord?
-        cord = capture_with_info(*app.cord(version: version), raise_on_non_zero_exit: false).strip
-        if cord.present?
-          execute *app.cut_cord(cord)
-          Kamal::Cli::Healthcheck::Poller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+      execute *app.stop(version: version), raise_on_non_zero_exit: false
+      execute *app.clean_up_assets if assets?
+      execute *app.clean_up_error_pages if KAMAL.config.error_pages_path
+    end
+
+    def release_barrier
+      if barrier.open
+        info "First #{KAMAL.primary_role} container is healthy on #{host}, booting any other roles"
+      end
+    end
+
+    def wait_at_barrier
+      info "Waiting for the first healthy #{KAMAL.primary_role} container before booting #{role} on #{host}..."
+      barrier.wait
+      info "First #{KAMAL.primary_role} container is healthy, booting #{role} on #{host}..."
+    rescue Kamal::Cli::Healthcheck::Error
+      info "First #{KAMAL.primary_role} container is unhealthy, not booting #{role} on #{host}"
+      raise
+    end
+
+    def close_barrier
+      if barrier.close
+        info "First #{KAMAL.primary_role} container is unhealthy on #{host}, not booting any other roles"
+        begin
+          error capture_with_info(*app.logs(container_id: app.container_id_for_version(version)))
+          error capture_with_info(*app.container_health_log(version: version))
+        rescue SSHKit::Command::Failed
+          error "Could not fetch logs for #{version}"
         end
       end
+    end
 
-      execute *app.stop(version: version), raise_on_non_zero_exit: false
+    def barrier_role?
+      role == KAMAL.primary_role
+    end
 
-      execute *app.clean_up_assets if assets?
+    def app
+      @app ||= KAMAL.app(role: role, host: host)
+    end
+
+    def auditor
+      @auditor = KAMAL.auditor(role: role)
+    end
+
+    def audit(message)
+      execute *auditor.record(message), verbosity: :debug
+    end
+
+    def gatekeeper?
+      barrier && barrier_role?
+    end
+
+    def queuer?
+      barrier && !barrier_role?
     end
 end

lib/kamal/cli/app/error_pages.rb

@@ -0,0 +1,33 @@
+class Kamal::Cli::App::ErrorPages
+  ERROR_PAGES_GLOB = "{4??.html,5??.html}"
+
+  attr_reader :host, :sshkit
+  delegate :upload!, :execute, to: :sshkit
+
+  def initialize(host, sshkit)
+    @host = host
+    @sshkit = sshkit
+  end
+
+  def run
+    if KAMAL.config.error_pages_path
+      with_error_pages_tmpdir do |local_error_pages_dir|
+        execute *KAMAL.app.create_error_pages_directory
+        upload! local_error_pages_dir, KAMAL.config.proxy_boot.error_pages_directory, mode: "0700", recursive: true
+      end
+    end
+  end
+
+  private
+    def with_error_pages_tmpdir
+      Dir.mktmpdir("kamal-error-pages") do |tmpdir|
+        error_pages_dir = File.join(tmpdir, KAMAL.config.version)
+        FileUtils.mkdir(error_pages_dir)
+
+        if (files = Dir[File.join(KAMAL.config.error_pages_path, ERROR_PAGES_GLOB)]).any?
+          FileUtils.cp(files, error_pages_dir)
+          yield error_pages_dir
+        end
+      end
+    end
+end

lib/kamal/cli/app/ssl_certificates.rb

@@ -0,0 +1,28 @@
+class Kamal::Cli::App::SslCertificates
+  attr_reader :host, :role, :sshkit
+  delegate :execute, :info, :upload!, to: :sshkit
+
+  def initialize(host, role, sshkit)
+    @host = host
+    @role = role
+    @sshkit = sshkit
+  end
+
+  def run
+    if role.running_proxy? && role.proxy.custom_ssl_certificate?
+      info "Writing SSL certificates for #{role.name} on #{host}"
+      execute *app.create_ssl_directory
+      if cert_content = role.proxy.certificate_pem_content
+        upload!(StringIO.new(cert_content), role.proxy.host_tls_cert, mode: "0644")
+      end
+      if key_content = role.proxy.private_key_pem_content
+        upload!(StringIO.new(key_content), role.proxy.host_tls_key, mode: "0644")
+      end
+    end
+  end
+
+  private
+    def app
+      @app ||= KAMAL.app(role: role, host: host)
+    end
+end

lib/kamal/cli/base.rb

@@ -1,5 +1,4 @@
 require "thor"
-require "dotenv"
 require "kamal/sshkit_with_ext"
 
 module Kamal::Cli
@@ -7,6 +6,7 @@ module Kamal::Cli
     include SSHKit::DSL
 
     def self.exit_on_failure?() true end
+    def self.dynamic_command_class() Kamal::Cli::Alias::Command end
 
     class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging"
     class_option :quiet, type: :boolean, aliases: "-q", desc: "Minimal logging"
@@ -22,33 +22,24 @@ module Kamal::Cli
 
     class_option :skip_hooks, aliases: "-H", type: :boolean, default: false, desc: "Don't run hooks"
 
-    def initialize(*)
-      super
-      @original_env = ENV.to_h.dup
-      load_envs
-      initialize_commander(options_with_subcommand_class_options)
+    def initialize(args = [], local_options = {}, config = {})
+      if config[:current_command].is_a?(Kamal::Cli::Alias::Command)
+        # When Thor generates a dynamic command, it doesn't attempt to parse the arguments.
+        # For our purposes, it means the arguments are passed in args rather than local_options.
+        super([], args, config)
+      else
+        super
+      end
+
+      initialize_commander unless KAMAL.configured?
     end
 
     private
-      def load_envs
-        if destination = options[:destination]
-          Dotenv.load(".env.#{destination}", ".env")
-        else
-          Dotenv.load(".env")
-        end
-      end
-
-      def reload_envs
-        ENV.clear
-        ENV.update(@original_env)
-        load_envs
-      end
-
       def options_with_subcommand_class_options
         options.merge(@_initializer.last[:class_options] || {})
       end
 
-      def initialize_commander(options)
+      def initialize_commander
         KAMAL.tap do |commander|
           if options[:verbose]
             ENV["VERBOSE"] = "1" # For backtraces via cli/start
@@ -79,28 +70,25 @@ module Kamal::Cli
         puts "  Finished all in #{sprintf("%.1f seconds", runtime)}"
       end
 
-      def mutating
-        return yield if KAMAL.holding_lock?
-
-        run_hook "pre-connect"
-
-        ensure_run_and_locks_directory
-
-        acquire_lock
-
-        begin
+      def with_lock
+        if KAMAL.holding_lock?
           yield
-        rescue
-          if KAMAL.hold_lock_on_error?
-            error "  \e[31mDeploy lock was not released\e[0m"
-          else
-            release_lock
+        else
+          acquire_lock
+
+          begin
+            yield
+          rescue
+            begin
+              release_lock
+            rescue => e
+              say "Error releasing the deploy lock: #{e.message}", :red
+            end
+            raise
           end
 
-          raise
+          release_lock
         end
-
-        release_lock
       end
 
       def confirming(question)
@@ -114,6 +102,8 @@ module Kamal::Cli
       end
 
       def acquire_lock
+        ensure_run_directory
+
         raise_if_locked do
           say "Acquiring the deploy lock...", :magenta
           on(KAMAL.primary_host) { execute *KAMAL.lock.acquire("Automatic deploy lock", KAMAL.config.version), verbosity: :debug }
@@ -141,26 +131,37 @@ module Kamal::Cli
         end
       end
 
-      def hold_lock_on_error
-        if KAMAL.hold_lock_on_error?
-          yield
-        else
-          KAMAL.hold_lock_on_error = true
-          yield
-          KAMAL.hold_lock_on_error = false
+      def run_hook(hook, **extra_details)
+        if !options[:skip_hooks] && KAMAL.hook.hook_exists?(hook)
+          details = {
+            hosts: KAMAL.hosts.join(","),
+            roles: KAMAL.specific_roles&.join(","),
+            lock: KAMAL.holding_lock?.to_s,
+            command: command,
+            subcommand: subcommand
+          }.compact
+
+          say "Running the #{hook} hook...", :magenta
+          with_env KAMAL.hook.env(**details, **extra_details) do
+            run_locally do
+              execute *KAMAL.hook.run(hook)
+            end
+          rescue SSHKit::Command::Failed => e
+            raise HookError.new("Hook `#{hook}` failed:\n#{e.message}")
+          end
         end
       end
 
-      def run_hook(hook, **extra_details)
-        if !options[:skip_hooks] && KAMAL.hook.hook_exists?(hook)
-          details = { hosts: KAMAL.hosts.join(","), command: command, subcommand: subcommand }
+      def on(*args, &block)
+        pre_connect_if_required
 
-          say "Running the #{hook} hook...", :magenta
-          run_locally do
-            KAMAL.with_verbosity(:debug) { execute *KAMAL.hook.run(hook, **details, **extra_details) }
-          rescue SSHKit::Command::Failed
-            raise HookError.new("Hook `#{hook}` failed")
-          end
+        super
+      end
+
+      def pre_connect_if_required
+        if !KAMAL.connected?
+          run_hook "pre-connect"
+          KAMAL.connected = true
         end
       end
 
@@ -186,13 +187,36 @@ module Kamal::Cli
         instance_variable_get("@_invocations").first
       end
 
-      def ensure_run_and_locks_directory
+      def reset_invocation(cli_class)
+        instance_variable_get("@_invocations")[cli_class].pop
+      end
+
+      def ensure_run_directory
         on(KAMAL.hosts) do
           execute(*KAMAL.server.ensure_run_directory)
         end
+      end
 
-        on(KAMAL.primary_host) do
-          execute(*KAMAL.lock.ensure_locks_directory)
+      def with_env(env)
+        current_env = ENV.to_h.dup
+        ENV.update(env)
+        yield
+      ensure
+        ENV.clear
+        ENV.update(current_env)
+      end
+
+      def ensure_docker_installed
+        run_locally do
+          begin
+            execute *KAMAL.builder.ensure_docker_installed
+          rescue SSHKit::Command::Failed => e
+            error = e.message =~ /command not found/ ?
+              "Docker is not installed locally" :
+              "Docker buildx plugin is not installed locally"
+
+            raise DependencyError, error
+          end
         end
       end
   end

lib/kamal/cli/build.rb

@@ -5,74 +5,99 @@ class Kamal::Cli::Build < Kamal::Cli::Base
 
   desc "deliver", "Build app and push app image to registry then pull image on servers"
   def deliver
-    mutating do
-      push
-      pull
-    end
+    invoke :push
+    invoke :pull
   end
 
   desc "push", "Build and push app image to registry"
+  option :output, type: :string, default: "registry", banner: "export_type", desc: "Exported type for the build result, and may be any exported type supported by 'buildx --output'."
   def push
-    mutating do
-      cli = self
+    cli = self
 
-      verify_local_dependencies
-      run_hook "pre-build"
+    # Ensure pre-connect hooks run before the build, they may needed for a remote builder
+    # or the pre-build hooks.
+    pre_connect_if_required
 
-      if (uncommitted_changes = Kamal::Git.uncommitted_changes).present?
-        say "The following paths have uncommitted changes:\n #{uncommitted_changes}", :yellow
+    ensure_docker_installed
+    login_to_registry_locally
+
+    run_hook "pre-build"
+
+    uncommitted_changes = Kamal::Git.uncommitted_changes
+
+    if KAMAL.config.builder.git_clone?
+      if uncommitted_changes.present?
+        say "Building from a local git clone, so ignoring these uncommitted changes:\n #{uncommitted_changes}", :yellow
       end
 
       run_locally do
-        begin
-          KAMAL.with_verbosity(:debug) do
-            execute *KAMAL.builder.push
-          end
-        rescue SSHKit::Command::Failed => e
-          if e.message =~ /(no builder)|(no such file or directory)/
-            error "Missing compatible builder, so creating a new one first"
+        Clone.new(self).prepare
+      end
+    elsif uncommitted_changes.present?
+      say "Building with uncommitted changes:\n #{uncommitted_changes}", :yellow
+    end
 
-            if cli.create
-              KAMAL.with_verbosity(:debug) { execute *KAMAL.builder.push }
+    with_env(KAMAL.config.builder.secrets) do
+      run_locally do
+        begin
+          execute *KAMAL.builder.inspect_builder
+        rescue SSHKit::Command::Failed => e
+          if e.message =~ /(context not found|no builder|no compatible builder|does not exist)/
+            warn "Missing compatible builder, so creating a new one first"
+            begin
+              cli.remove
+            rescue SSHKit::Command::Failed
+              raise unless e.message =~ /(context not found|no builder|does not exist)/
             end
+            cli.create
           else
             raise
           end
         end
+
+        # Get the command here to ensure the Dir.chdir doesn't interfere with it
+        push = KAMAL.builder.push(cli.options[:output])
+
+        KAMAL.with_verbosity(:debug) do
+          Dir.chdir(KAMAL.config.builder.build_directory) { execute *push }
+        end
       end
     end
   end
 
   desc "pull", "Pull app image from registry onto servers"
   def pull
-    mutating do
-      on(KAMAL.hosts) do
-        execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
-        execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
-        execute *KAMAL.builder.pull
-        execute *KAMAL.builder.validate_image
+    login_to_registry_remotely unless KAMAL.registry.local?
+
+    forward_local_registry_port do
+      if (first_hosts = mirror_hosts).any?
+        #  Pull on a single host per mirror first to seed them
+        say "Pulling image on #{first_hosts.join(", ")} to seed the #{"mirror".pluralize(first_hosts.count)}...", :magenta
+        pull_on_hosts(first_hosts)
+        say "Pulling image on remaining hosts...", :magenta
+        pull_on_hosts(KAMAL.app_hosts - first_hosts)
+      else
+        pull_on_hosts(KAMAL.app_hosts)
       end
     end
   end
 
   desc "create", "Create a build setup"
   def create
-    mutating do
-      if (remote_host = KAMAL.config.builder.remote_host)
-        connect_to_remote_host(remote_host)
-      end
+    if (remote_host = KAMAL.config.builder.remote)
+      connect_to_remote_host(remote_host)
+    end
 
-      run_locally do
-        begin
-          debug "Using builder: #{KAMAL.builder.name}"
-          execute *KAMAL.builder.create
-        rescue SSHKit::Command::Failed => e
-          if e.message =~ /stderr=(.*)/
-            error "Couldn't create remote builder: #{$1}"
-            false
-          else
-            raise
-          end
+    run_locally do
+      begin
+        debug "Using builder: #{KAMAL.builder.name}"
+        execute *KAMAL.builder.create
+      rescue SSHKit::Command::Failed => e
+        if e.message =~ /stderr=(.*)/
+          error "Couldn't create remote builder: #{$1}"
+          false
+        else
+          raise
         end
       end
     end
@@ -80,11 +105,9 @@ class Kamal::Cli::Build < Kamal::Cli::Base
 
   desc "remove", "Remove build setup"
   def remove
-    mutating do
-      run_locally do
-        debug "Using builder: #{KAMAL.builder.name}"
-        execute *KAMAL.builder.remove
-      end
+    run_locally do
+      debug "Using builder: #{KAMAL.builder.name}"
+      execute *KAMAL.builder.remove
     end
   end
 
@@ -96,28 +119,102 @@ class Kamal::Cli::Build < Kamal::Cli::Base
     end
   end
 
-  private
-    def verify_local_dependencies
-      run_locally do
-        begin
-          execute *KAMAL.builder.ensure_local_dependencies_installed
-        rescue SSHKit::Command::Failed => e
-          build_error = e.message =~ /command not found/ ?
-            "Docker is not installed locally" :
-            "Docker buildx plugin is not installed locally"
+  desc "dev", "Build using the working directory, tag it as dirty, and push to local image store."
+  option :output, type: :string, default: "docker", banner: "export_type", desc: "Exported type for the build result, and may be any exported type supported by 'buildx --output'."
+  def dev
+    cli = self
 
-          raise BuildError, build_error
+    ensure_docker_installed
+
+    docker_included_files = Set.new(Kamal::Docker.included_files)
+    git_uncommitted_files = Set.new(Kamal::Git.uncommitted_files)
+    git_untracked_files = Set.new(Kamal::Git.untracked_files)
+
+    docker_uncommitted_files = docker_included_files & git_uncommitted_files
+    if docker_uncommitted_files.any?
+      say "WARNING: Files with uncommitted changes will be present in the dev container:", :yellow
+      docker_uncommitted_files.sort.each { |f| say "  #{f}", :yellow }
+      say
+    end
+
+    docker_untracked_files = docker_included_files & git_untracked_files
+    if docker_untracked_files.any?
+      say "WARNING: Untracked files will be present in the dev container:", :yellow
+      docker_untracked_files.sort.each { |f| say "  #{f}", :yellow }
+      say
+    end
+
+    with_env(KAMAL.config.builder.secrets) do
+      run_locally do
+        build = KAMAL.builder.push(cli.options[:output], tag_as_dirty: true)
+        KAMAL.with_verbosity(:debug) do
+          execute(*build)
         end
       end
     end
+  end
 
+  private
     def connect_to_remote_host(remote_host)
       remote_uri = URI.parse(remote_host)
       if remote_uri.scheme == "ssh"
-        options = { user: remote_uri.user, port: remote_uri.port }.compact
-        on(remote_uri.host, options) do
+        host = SSHKit::Host.new(
+          hostname: remote_uri.host,
+          ssh_options: { user: remote_uri.user, port: remote_uri.port }.compact
+        )
+        on(host, options) do
           execute "true"
         end
       end
     end
+
+    def mirror_hosts
+      if KAMAL.app_hosts.many?
+        mirror_hosts = Concurrent::Hash.new
+        on(KAMAL.app_hosts) do |host|
+          first_mirror = capture_with_info(*KAMAL.builder.first_mirror).strip.presence
+          mirror_hosts[first_mirror] ||= host.to_s if first_mirror
+        rescue SSHKit::Command::Failed => e
+          raise unless e.message =~ /error calling index: reflect: slice index out of range/
+        end
+        mirror_hosts.values
+      else
+        []
+      end
+    end
+
+    def pull_on_hosts(hosts)
+      on(hosts) do
+        execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
+        execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
+        execute *KAMAL.builder.pull
+        execute *KAMAL.builder.validate_image
+      end
+    end
+
+    def login_to_registry_locally
+      run_locally do
+        if KAMAL.registry.local?
+          execute *KAMAL.registry.setup
+        else
+          execute *KAMAL.registry.login
+        end
+      end
+    end
+
+    def login_to_registry_remotely
+      on(KAMAL.app_hosts) do
+        execute *KAMAL.registry.login
+      end
+    end
+
+    def forward_local_registry_port(&block)
+      if KAMAL.config.registry.local?
+        Kamal::Cli::PortForwarding.
+          new(KAMAL.hosts, KAMAL.config.registry.local_port).
+          forward(&block)
+      else
+        yield
+      end
+    end
 end

lib/kamal/cli/build/clone.rb

@@ -0,0 +1,61 @@
+require "uri"
+
+class Kamal::Cli::Build::Clone
+  attr_reader :sshkit
+  delegate :info, :error, :execute, :capture_with_info, to: :sshkit
+
+  def initialize(sshkit)
+    @sshkit = sshkit
+  end
+
+  def prepare
+    begin
+      clone_repo
+    rescue SSHKit::Command::Failed => e
+      if e.message =~ /already exists and is not an empty directory/
+        reset
+      else
+        raise Kamal::Cli::Build::BuildError, "Failed to clone repo: #{e.message}"
+      end
+    end
+
+    validate!
+  rescue Kamal::Cli::Build::BuildError => e
+    error "Error preparing clone: #{e.message}, deleting and retrying..."
+
+    FileUtils.rm_rf KAMAL.config.builder.clone_directory
+    clone_repo
+    validate!
+  end
+
+  private
+    def clone_repo
+      info "Cloning repo into build directory `#{KAMAL.config.builder.build_directory}`..."
+
+      FileUtils.mkdir_p KAMAL.config.builder.clone_directory
+      execute *KAMAL.builder.clone
+    end
+
+    def reset
+      info "Resetting local clone as `#{KAMAL.config.builder.build_directory}` already exists..."
+
+      KAMAL.builder.clone_reset_steps.each { |step| execute *step }
+    rescue SSHKit::Command::Failed => e
+      raise Kamal::Cli::Build::BuildError, "Failed to clone repo: #{e.message}"
+    end
+
+    def validate!
+      status = capture_with_info(*KAMAL.builder.clone_status).strip
+
+      unless status.empty?
+        raise Kamal::Cli::Build::BuildError, "Clone in #{KAMAL.config.builder.build_directory} is dirty, #{status}"
+      end
+
+      revision = capture_with_info(*KAMAL.builder.clone_revision).strip
+      if revision != Kamal::Git.revision
+        raise Kamal::Cli::Build::BuildError, "Clone in #{KAMAL.config.builder.build_directory} is not on the correct revision, expected `#{Kamal::Git.revision}` but got `#{revision}`"
+      end
+    rescue SSHKit::Command::Failed => e
+      raise Kamal::Cli::Build::BuildError, "Failed to validate clone: #{e.message}"
+    end
+end

lib/kamal/cli/env.rb

@@ -1,54 +0,0 @@
-require "tempfile"
-
-class Kamal::Cli::Env < Kamal::Cli::Base
-  desc "push", "Push the env file to the remote hosts"
-  def push
-    mutating do
-      on(KAMAL.hosts) do
-        execute *KAMAL.auditor.record("Pushed env files"), verbosity: :debug
-
-        KAMAL.roles_on(host).each do |role|
-          execute *KAMAL.app(role: role).make_env_directory
-          upload! role.env.secrets_io, role.env.secrets_file, mode: 400
-        end
-      end
-
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.traefik.make_env_directory
-        upload! KAMAL.traefik.env.secrets_io, KAMAL.traefik.env.secrets_file, mode: 400
-      end
-
-      on(KAMAL.accessory_hosts) do
-        KAMAL.accessories_on(host).each do |accessory|
-          accessory_config = KAMAL.config.accessory(accessory)
-          execute *KAMAL.accessory(accessory).make_env_directory
-          upload! accessory_config.env.secrets_io, accessory_config.env.secrets_file, mode: 400
-        end
-      end
-    end
-  end
-
-  desc "delete", "Delete the env file from the remote hosts"
-  def delete
-    mutating do
-      on(KAMAL.hosts) do
-        execute *KAMAL.auditor.record("Deleted env files"), verbosity: :debug
-
-        KAMAL.roles_on(host).each do |role|
-          execute *KAMAL.app(role: role).remove_env_file
-        end
-      end
-
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.traefik.remove_env_file
-      end
-
-      on(KAMAL.accessory_hosts) do
-        KAMAL.accessories_on(host).each do |accessory|
-          accessory_config = KAMAL.config.accessory(accessory)
-          execute *KAMAL.accessory(accessory).remove_env_file
-        end
-      end
-    end
-  end
-end

lib/kamal/cli/healthcheck.rb

@@ -1,21 +0,0 @@
-class Kamal::Cli::Healthcheck < Kamal::Cli::Base
-  default_command :perform
-
-  desc "perform", "Health check current app version"
-  def perform
-    raise "The primary host is not configured to run Traefik" unless KAMAL.config.role(KAMAL.config.primary_role).running_traefik?
-    on(KAMAL.primary_host) do
-      begin
-        execute *KAMAL.healthcheck.run
-        Poller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
-      rescue Poller::HealthcheckError => e
-        error capture_with_info(*KAMAL.healthcheck.logs)
-        error capture_with_pretty_json(*KAMAL.healthcheck.container_health_log)
-        raise
-      ensure
-        execute *KAMAL.healthcheck.stop, raise_on_non_zero_exit: false
-        execute *KAMAL.healthcheck.remove, raise_on_non_zero_exit: false
-      end
-    end
-  end
-end

lib/kamal/cli/healthcheck/barrier.rb

@@ -0,0 +1,33 @@
+require "concurrent/ivar"
+
+class Kamal::Cli::Healthcheck::Barrier
+  def initialize
+    @ivar = Concurrent::IVar.new
+  end
+
+  def close
+    set(false)
+  end
+
+  def open
+    set(true)
+  end
+
+  def wait
+    unless opened?
+      raise Kamal::Cli::Healthcheck::Error.new("Halted at barrier")
+    end
+  end
+
+  private
+    def opened?
+      @ivar.value
+    end
+
+    def set(value)
+      @ivar.set(value)
+      true
+    rescue Concurrent::MultipleAssignmentError
+      false
+    end
+end

lib/kamal/cli/healthcheck/error.rb

@@ -0,0 +1,2 @@
+class Kamal::Cli::Healthcheck::Error < StandardError
+end

lib/kamal/cli/healthcheck/poller.rb

@@ -1,27 +1,30 @@
 module Kamal::Cli::Healthcheck::Poller
   extend self
 
-  TRAEFIK_UPDATE_DELAY = 5
-
-  class HealthcheckError < StandardError; end
-
-  def wait_for_healthy(pause_after_ready: false, &block)
+  def wait_for_healthy(role, &block)
     attempt = 1
-    max_attempts = KAMAL.config.healthcheck["max_attempts"]
+    timeout_at = Time.now + KAMAL.config.deploy_timeout
+    readiness_delay = KAMAL.config.readiness_delay
 
     begin
-      case status = block.call
-      when "healthy"
-        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
-      when "running" # No health check configured
-        sleep KAMAL.config.readiness_delay if pause_after_ready
-      else
-        raise HealthcheckError, "container not ready (#{status})"
+      status = block.call
+
+      if status == "running"
+        # Wait for the readiness delay and confirm it is still running
+        if readiness_delay > 0
+          info "Container is running, waiting for readiness delay of #{readiness_delay} seconds"
+          sleep readiness_delay
+          status = block.call
+        end
       end
-    rescue HealthcheckError => e
-      if attempt <= max_attempts
-        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-        sleep attempt
+
+      unless %w[ running healthy ].include?(status)
+        raise Kamal::Cli::Healthcheck::Error, "container not ready after #{KAMAL.config.deploy_timeout} seconds (#{status})"
+      end
+    rescue Kamal::Cli::Healthcheck::Error => e
+      time_left = timeout_at - Time.now
+      if time_left > 0
+        sleep [ attempt, time_left ].min
         attempt += 1
         retry
       else
@@ -32,31 +35,6 @@ module Kamal::Cli::Healthcheck::Poller
     info "Container is healthy!"
   end
 
-  def wait_for_unhealthy(pause_after_ready: false, &block)
-    attempt = 1
-    max_attempts = KAMAL.config.healthcheck["max_attempts"]
-
-    begin
-      case status = block.call
-      when "unhealthy"
-        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
-      else
-        raise HealthcheckError, "container not unhealthy (#{status})"
-      end
-    rescue HealthcheckError => e
-      if attempt <= max_attempts
-        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-        sleep attempt
-        attempt += 1
-        retry
-      else
-        raise
-      end
-    end
-
-    info "Container is unhealthy!"
-  end
-
   private
     def info(message)
       SSHKit.config.output.info(message)

lib/kamal/cli/lock.rb

@@ -3,7 +3,6 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   def status
     handle_missing_lock do
       on(KAMAL.primary_host) do
-        execute *KAMAL.server.ensure_run_directory
         puts capture_with_debug(*KAMAL.lock.status)
       end
     end
@@ -13,9 +12,10 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   option :message, aliases: "-m", type: :string, desc: "A lock message", required: true
   def acquire
     message = options[:message]
+    ensure_run_directory
+
     raise_if_locked do
       on(KAMAL.primary_host) do
-        execute *KAMAL.server.ensure_run_directory
         execute *KAMAL.lock.acquire(message, KAMAL.config.version), verbosity: :debug
       end
       say "Acquired the deploy lock"
@@ -26,7 +26,6 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   def release
     handle_missing_lock do
       on(KAMAL.primary_host) do
-        execute *KAMAL.server.ensure_run_directory
         execute *KAMAL.lock.release, verbosity: :debug
       end
       say "Released the deploy lock"

lib/kamal/cli/main.rb

@@ -3,48 +3,38 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def setup
     print_runtime do
-      mutating do
+      with_lock do
         invoke_options = deploy_options
 
         say "Ensure Docker is installed...", :magenta
         invoke "kamal:cli:server:bootstrap", [], invoke_options
 
-        say "Push env files...", :magenta
-        invoke "kamal:cli:env:push", [], invoke_options
-
-        invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options
-        deploy
+        deploy(boot_accessories: true)
       end
     end
   end
 
   desc "deploy", "Deploy app to servers"
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
-  def deploy
+  def deploy(boot_accessories: false)
     runtime = print_runtime do
-      mutating do
-        invoke_options = deploy_options
+      invoke_options = deploy_options
 
-        say "Log into image registry...", :magenta
-        invoke "kamal:cli:registry:login", [], invoke_options
+      if options[:skip_push]
+        say "Pull app image...", :magenta
+        invoke "kamal:cli:build:pull", [], invoke_options
+      else
+        say "Build and push app image...", :magenta
+        invoke "kamal:cli:build:deliver", [], invoke_options
+      end
 
-        if options[:skip_push]
-          say "Pull app image...", :magenta
-          invoke "kamal:cli:build:pull", [], invoke_options
-        else
-          say "Build and push app image...", :magenta
-          invoke "kamal:cli:build:deliver", [], invoke_options
-        end
+      with_lock do
+        run_hook "pre-deploy", secrets: true
 
-        run_hook "pre-deploy"
+        say "Ensure kamal-proxy is running...", :magenta
+        invoke "kamal:cli:proxy:boot", [], invoke_options
 
-        say "Ensure Traefik is running...", :magenta
-        invoke "kamal:cli:traefik:boot", [], invoke_options
-
-        if KAMAL.config.role(KAMAL.config.primary_role).running_traefik?
-          say "Ensure app can pass healthcheck...", :magenta
-          invoke "kamal:cli:healthcheck:perform", [], invoke_options
-        end
+        invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options if boot_accessories
 
         say "Detect stale containers...", :magenta
         invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
@@ -56,28 +46,25 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       end
     end
 
-    run_hook "post-deploy", runtime: runtime.round
+    run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s
   end
 
-  desc "redeploy", "Deploy app to servers without bootstrapping servers, starting Traefik, pruning, and registry login"
+  desc "redeploy", "Deploy app to servers without bootstrapping servers, starting kamal-proxy and pruning"
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def redeploy
     runtime = print_runtime do
-      mutating do
-        invoke_options = deploy_options
+      invoke_options = deploy_options
 
-        if options[:skip_push]
-          say "Pull app image...", :magenta
-          invoke "kamal:cli:build:pull", [], invoke_options
-        else
-          say "Build and push app image...", :magenta
-          invoke "kamal:cli:build:deliver", [], invoke_options
-        end
+      if options[:skip_push]
+        say "Pull app image...", :magenta
+        invoke "kamal:cli:build:pull", [], invoke_options
+      else
+        say "Build and push app image...", :magenta
+        invoke "kamal:cli:build:deliver", [], invoke_options
+      end
 
-        run_hook "pre-deploy"
-
-        say "Ensure app can pass healthcheck...", :magenta
-        invoke "kamal:cli:healthcheck:perform", [], invoke_options
+      with_lock do
+        run_hook "pre-deploy", secrets: true
 
         say "Detect stale containers...", :magenta
         invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
@@ -86,21 +73,21 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       end
     end
 
-    run_hook "post-deploy", runtime: runtime.round
+    run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s
   end
 
   desc "rollback [VERSION]", "Rollback app to VERSION"
   def rollback(version)
     rolled_back = false
     runtime = print_runtime do
-      mutating do
+      with_lock do
         invoke_options = deploy_options
 
         KAMAL.config.version = version
         old_version = nil
 
         if container_available?(version)
-          run_hook "pre-deploy"
+          run_hook "pre-deploy", secrets: true
 
           invoke "kamal:cli:app:boot", [], invoke_options.merge(version: version)
           rolled_back = true
@@ -110,12 +97,12 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       end
     end
 
-    run_hook "post-deploy", runtime: runtime.round if rolled_back
+    run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s if rolled_back
   end
 
   desc "details", "Show details about all containers"
   def details
-    invoke "kamal:cli:traefik:details"
+    invoke "kamal:cli:proxy:details"
     invoke "kamal:cli:app:details"
     invoke "kamal:cli:accessory:details", [ "all" ]
   end
@@ -134,7 +121,19 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     end
   end
 
-  desc "init", "Create config stub in config/deploy.yml and env stub in .env"
+  desc "docs [SECTION]", "Show Kamal configuration documentation"
+  def docs(section = nil)
+    case section
+    when NilClass
+      puts Kamal::Configuration.validation_doc
+    else
+      puts Kamal::Configuration.const_get(section.titlecase.to_sym).validation_doc
+    end
+  rescue NameError
+    puts "No documentation found for #{section}"
+  end
+
+  desc "init", "Create config stub in config/deploy.yml and secrets stub in .kamal"
   option :bundle, type: :boolean, default: false, desc: "Add Kamal to the Gemfile and create a bin/kamal binstub"
   def init
     require "fileutils"
@@ -147,9 +146,10 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       puts "Created configuration file in config/deploy.yml"
     end
 
-    unless (deploy_file = Pathname.new(File.expand_path(".env"))).exist?
-      FileUtils.cp_r Pathname.new(File.expand_path("templates/template.env", __dir__)), deploy_file
-      puts "Created .env file"
+    unless (secrets_file = Pathname.new(File.expand_path(".kamal/secrets"))).exist?
+      FileUtils.mkdir_p secrets_file.dirname
+      FileUtils.cp_r Pathname.new(File.expand_path("templates/secrets", __dir__)), secrets_file
+      puts "Created .kamal/secrets file"
     end
 
     unless (hooks_dir = Pathname.new(File.expand_path(".kamal/hooks"))).exist?
@@ -174,34 +174,46 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     end
   end
 
-  desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)"
-  option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip .env file push"
-  def envify
-    if destination = options[:destination]
-      env_template_path = ".env.#{destination}.erb"
-      env_path          = ".env.#{destination}"
-    else
-      env_template_path = ".env.erb"
-      env_path          = ".env"
-    end
-
-    File.write(env_path, ERB.new(File.read(env_template_path), trim_mode: "-").result, perm: 0600)
-
-    unless options[:skip_push]
-      reload_envs
-      invoke "kamal:cli:env:push", options
+  desc "remove", "Remove kamal-proxy, app, accessories, and registry session from servers"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  def remove
+    confirming "This will remove all containers and images. Are you sure?" do
+      with_lock do
+        invoke "kamal:cli:app:remove", [], options.without(:confirmed)
+        invoke "kamal:cli:proxy:remove", [], options.without(:confirmed)
+        invoke "kamal:cli:accessory:remove", [ "all" ], options
+        invoke "kamal:cli:registry:remove", [], options.without(:confirmed).merge(skip_local: true)
+      end
     end
   end
 
-  desc "remove", "Remove Traefik, app, accessories, and registry session from servers"
+  desc "upgrade", "Upgrade from Kamal 1.x to 2.0"
   option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
-  def remove
-    mutating do
-      confirming "This will remove all containers and images. Are you sure?" do
-        invoke "kamal:cli:traefik:remove", [], options.without(:confirmed)
-        invoke "kamal:cli:app:remove", [], options.without(:confirmed)
-        invoke "kamal:cli:accessory:remove", [ "all" ], options
-        invoke "kamal:cli:registry:logout", [], options.without(:confirmed)
+  option :rolling, type: :boolean, default: false, desc: "Upgrade one host at a time"
+  def upgrade
+    confirming "This will replace Traefik with kamal-proxy and restart all accessories" do
+      with_lock do
+        if options[:rolling]
+          KAMAL.hosts.each do |host|
+            KAMAL.with_specific_hosts(host) do
+              say "Upgrading #{host}...", :magenta
+              if KAMAL.app_hosts.include?(host)
+                invoke "kamal:cli:proxy:upgrade", [], options.merge(confirmed: true, rolling: false)
+                reset_invocation(Kamal::Cli::Proxy)
+              end
+              if KAMAL.accessory_hosts.include?(host)
+                invoke "kamal:cli:accessory:upgrade", [ "all" ], options.merge(confirmed: true, rolling: false)
+                reset_invocation(Kamal::Cli::Accessory)
+              end
+              say "Upgraded #{host}", :magenta
+            end
+          end
+        else
+          say "Upgrading all hosts...", :magenta
+          invoke "kamal:cli:proxy:upgrade", [], options.merge(confirmed: true)
+          invoke "kamal:cli:accessory:upgrade", [ "all" ], options.merge(confirmed: true)
+          say "Upgraded all hosts", :magenta
+        end
       end
     end
   end
@@ -220,37 +232,34 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   desc "build", "Build application image"
   subcommand "build", Kamal::Cli::Build
 
-  desc "env", "Manage environment files"
-  subcommand "env", Kamal::Cli::Env
-
-  desc "healthcheck", "Healthcheck application"
-  subcommand "healthcheck", Kamal::Cli::Healthcheck
-
   desc "lock", "Manage the deploy lock"
   subcommand "lock", Kamal::Cli::Lock
 
+  desc "proxy", "Manage kamal-proxy"
+  subcommand "proxy", Kamal::Cli::Proxy
+
   desc "prune", "Prune old application images and containers"
   subcommand "prune", Kamal::Cli::Prune
 
   desc "registry", "Login and -out of the image registry"
   subcommand "registry", Kamal::Cli::Registry
 
+  desc "secrets", "Helpers for extracting secrets"
+  subcommand "secrets", Kamal::Cli::Secrets
+
   desc "server", "Bootstrap servers with curl and Docker"
   subcommand "server", Kamal::Cli::Server
 
-  desc "traefik", "Manage Traefik load balancer"
-  subcommand "traefik", Kamal::Cli::Traefik
-
   private
     def container_available?(version)
       begin
-        on(KAMAL.hosts) do
+        on(KAMAL.app_hosts) do
           KAMAL.roles_on(host).each do |role|
-            container_id = capture_with_info(*KAMAL.app(role: role).container_id_for_version(version))
+            container_id = capture_with_info(*KAMAL.app(role: role, host: host).container_id_for_version(version))
             raise "Container not found" unless container_id.present?
           end
         end
-      rescue SSHKit::Runner::ExecuteError => e
+      rescue SSHKit::Runner::ExecuteError, SSHKit::Runner::MultipleExecuteError => e
         if e.message =~ /Container not found/
           say "Error looking for container version #{version}: #{e.message}"
           return false

lib/kamal/cli/port_forwarding.rb

@@ -0,0 +1,42 @@
+class Kamal::Cli::PortForwarding
+	attr_reader :hosts, :port
+
+	def initialize(hosts, port)
+		@hosts = hosts
+		@port = port
+	end
+
+	def forward
+		@done = false
+		forward_ports
+
+		yield
+	ensure
+		stop
+	end
+
+	private
+
+	def stop
+		@done = true
+		@threads.to_a.each(&:join)
+	end
+
+	def forward_ports
+		@threads = hosts.map do |host|
+			Thread.new do
+				Net::SSH.start(host, KAMAL.config.ssh.user) do |ssh|
+					ssh.forward.remote(port, "127.0.0.1", port)
+					ssh.loop(0.1) do
+						if @done
+							ssh.forward.cancel_remote(port)
+							break
+						else
+							true
+						end
+					end
+				end
+			end
+		end
+	end
+end

lib/kamal/cli/proxy.rb

@@ -0,0 +1,290 @@
+class Kamal::Cli::Proxy < Kamal::Cli::Base
+  desc "boot", "Boot proxy on servers"
+  def boot
+    with_lock do
+      on(KAMAL.hosts) do |host|
+        execute *KAMAL.docker.create_network
+      rescue SSHKit::Command::Failed => e
+        raise unless e.message.include?("already exists")
+      end
+
+      on(KAMAL.proxy_hosts) do |host|
+        execute *KAMAL.registry.login
+
+        version = capture_with_info(*KAMAL.proxy.version).strip.presence
+
+        if version && Kamal::Utils.older_version?(version, Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION)
+          raise "kamal-proxy version #{version} is too old, run `kamal proxy reboot` in order to update to at least #{Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION}"
+        end
+        execute *KAMAL.proxy.ensure_apps_config_directory
+        execute *KAMAL.proxy.start_or_run
+      end
+    end
+  end
+
+  desc "boot_config <set|get|reset>", "Manage kamal-proxy boot configuration"
+  option :publish, type: :boolean, default: true, desc: "Publish the proxy ports on the host"
+  option :publish_host_ip, type: :string, repeatable: true, default: nil, desc: "Host IP address to bind HTTP/HTTPS traffic to. Defaults to all interfaces"
+  option :http_port, type: :numeric, default: Kamal::Configuration::Proxy::Boot::DEFAULT_HTTP_PORT, desc: "HTTP port to publish on the host"
+  option :https_port, type: :numeric, default: Kamal::Configuration::Proxy::Boot::DEFAULT_HTTPS_PORT, desc: "HTTPS port to publish on the host"
+  option :log_max_size, type: :string, default: Kamal::Configuration::Proxy::Boot::DEFAULT_LOG_MAX_SIZE, desc: "Max size of proxy logs"
+  option :registry, type: :string, default: nil, desc: "Registry to use for the proxy image"
+  option :repository, type: :string, default: nil, desc: "Repository for the proxy image"
+  option :image_version, type: :string, default: nil, desc: "Version of the proxy to run"
+  option :metrics_port, type: :numeric, default: nil, desc: "Port to report prometheus metrics on"
+  option :debug, type: :boolean, default: false, desc: "Whether to run the proxy in debug mode"
+  option :docker_options, type: :array, default: [], desc: "Docker options to pass to the proxy container", banner: "option=value option2=value2"
+  def boot_config(subcommand)
+    proxy_boot_config = KAMAL.config.proxy_boot
+
+    case subcommand
+    when "set"
+      boot_options = [
+        *(proxy_boot_config.publish_args(options[:http_port], options[:https_port], options[:publish_host_ip]) if options[:publish]),
+        *(proxy_boot_config.logging_args(options[:log_max_size])),
+        *("--expose=#{options[:metrics_port]}" if options[:metrics_port]),
+        *options[:docker_options].map { |option| "--#{option}" }
+      ]
+
+      image = [
+        options[:registry].presence,
+        options[:repository].presence || proxy_boot_config.repository_name,
+        proxy_boot_config.image_name
+      ].compact.join("/")
+
+      image_version = options[:image_version]
+
+      run_command_options = { debug: options[:debug] || nil, "metrics-port": options[:metrics_port] }.compact
+      run_command = "kamal-proxy run #{Kamal::Utils.optionize(run_command_options).join(" ")}" if run_command_options.any?
+
+      on(KAMAL.proxy_hosts) do |host|
+        execute(*KAMAL.proxy.ensure_proxy_directory)
+        if boot_options != proxy_boot_config.default_boot_options
+          upload! StringIO.new(boot_options.join(" ")), proxy_boot_config.options_file
+        else
+          execute *KAMAL.proxy.reset_boot_options, raise_on_non_zero_exit: false
+        end
+
+        if image != proxy_boot_config.image_default
+          upload! StringIO.new(image), proxy_boot_config.image_file
+        else
+          execute *KAMAL.proxy.reset_image, raise_on_non_zero_exit: false
+        end
+
+        if image_version
+          upload! StringIO.new(image_version), proxy_boot_config.image_version_file
+        else
+          execute *KAMAL.proxy.reset_image_version, raise_on_non_zero_exit: false
+        end
+
+        if run_command
+          upload! StringIO.new(run_command), proxy_boot_config.run_command_file
+        else
+          execute *KAMAL.proxy.reset_run_command, raise_on_non_zero_exit: false
+        end
+      end
+    when "get"
+
+      on(KAMAL.proxy_hosts) do |host|
+        puts "Host #{host}: #{capture_with_info(*KAMAL.proxy.boot_config)}"
+      end
+    when "reset"
+      on(KAMAL.proxy_hosts) do |host|
+        execute *KAMAL.proxy.reset_boot_options, raise_on_non_zero_exit: false
+        execute *KAMAL.proxy.reset_image, raise_on_non_zero_exit: false
+        execute *KAMAL.proxy.reset_image_version, raise_on_non_zero_exit: false
+        execute *KAMAL.proxy.reset_run_command, raise_on_non_zero_exit: false
+      end
+    else
+      raise ArgumentError, "Unknown boot_config subcommand #{subcommand}"
+    end
+  end
+
+  desc "reboot", "Reboot proxy on servers (stop container, remove container, start new container)"
+  option :rolling, type: :boolean, default: false, desc: "Reboot proxy on hosts in sequence, rather than in parallel"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  def reboot
+    confirming "This will cause a brief outage on each host. Are you sure?" do
+      with_lock do
+        host_groups = options[:rolling] ? KAMAL.proxy_hosts : [ KAMAL.proxy_hosts ]
+        host_groups.each do |hosts|
+          host_list = Array(hosts).join(",")
+          run_hook "pre-proxy-reboot", hosts: host_list
+          on(hosts) do |host|
+            execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
+            execute *KAMAL.registry.login
+
+            "Stopping and removing kamal-proxy on #{host}, if running..."
+            execute *KAMAL.proxy.stop, raise_on_non_zero_exit: false
+            execute *KAMAL.proxy.remove_container
+            execute *KAMAL.proxy.ensure_apps_config_directory
+
+            execute *KAMAL.proxy.run
+          end
+          run_hook "post-proxy-reboot", hosts: host_list
+        end
+      end
+    end
+  end
+
+  desc "upgrade", "Upgrade to kamal-proxy on servers (stop container, remove container, start new container, reboot app)", hide: true
+  option :rolling, type: :boolean, default: false, desc: "Reboot proxy on hosts in sequence, rather than in parallel"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  def upgrade
+    invoke_options = { "version" => KAMAL.config.latest_tag }.merge(options)
+
+    confirming "This will cause a brief outage on each host. Are you sure?" do
+      host_groups = options[:rolling] ? KAMAL.hosts : [ KAMAL.hosts ]
+      host_groups.each do |hosts|
+        host_list = Array(hosts).join(",")
+        say "Upgrading proxy on #{host_list}...", :magenta
+        run_hook "pre-proxy-reboot", hosts: host_list
+        on(hosts) do |host|
+          execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
+          execute *KAMAL.registry.login
+
+          "Stopping and removing Traefik on #{host}, if running..."
+          execute *KAMAL.proxy.cleanup_traefik
+
+          "Stopping and removing kamal-proxy on #{host}, if running..."
+          execute *KAMAL.proxy.stop, raise_on_non_zero_exit: false
+          execute *KAMAL.proxy.remove_container
+          execute *KAMAL.proxy.remove_image
+        end
+
+        KAMAL.with_specific_hosts(hosts) do
+          invoke "kamal:cli:proxy:boot", [], invoke_options
+          reset_invocation(Kamal::Cli::Proxy)
+          invoke "kamal:cli:app:boot", [], invoke_options
+          reset_invocation(Kamal::Cli::App)
+          invoke "kamal:cli:prune:all", [], invoke_options
+          reset_invocation(Kamal::Cli::Prune)
+        end
+
+        run_hook "post-proxy-reboot", hosts: host_list
+        say "Upgraded proxy on #{host_list}", :magenta
+      end
+    end
+  end
+
+  desc "start", "Start existing proxy container on servers"
+  def start
+    with_lock do
+      on(KAMAL.proxy_hosts) do |host|
+        execute *KAMAL.auditor.record("Started proxy"), verbosity: :debug
+        execute *KAMAL.proxy.start
+      end
+    end
+  end
+
+  desc "stop", "Stop existing proxy container on servers"
+  def stop
+    with_lock do
+      on(KAMAL.proxy_hosts) do |host|
+        execute *KAMAL.auditor.record("Stopped proxy"), verbosity: :debug
+        execute *KAMAL.proxy.stop, raise_on_non_zero_exit: false
+      end
+    end
+  end
+
+  desc "restart", "Restart existing proxy container on servers"
+  def restart
+    with_lock do
+      stop
+      start
+    end
+  end
+
+  desc "details", "Show details about proxy container from servers"
+  def details
+    on(KAMAL.proxy_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.proxy.info), type: "Proxy" }
+  end
+
+  desc "logs", "Show log lines from proxy on servers"
+  option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
+  option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server"
+  option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
+  option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
+  option :skip_timestamps, type: :boolean, aliases: "-T", desc: "Skip appending timestamps to logging output"
+  def logs
+    grep = options[:grep]
+    timestamps = !options[:skip_timestamps]
+
+    if options[:follow]
+      run_locally do
+        info "Following logs on #{KAMAL.primary_host}..."
+        info KAMAL.proxy.follow_logs(host: KAMAL.primary_host, timestamps: timestamps, grep: grep)
+        exec KAMAL.proxy.follow_logs(host: KAMAL.primary_host, timestamps: timestamps, grep: grep)
+      end
+    else
+      since = options[:since]
+      lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
+
+      on(KAMAL.proxy_hosts) do |host|
+        puts_by_host host, capture(*KAMAL.proxy.logs(timestamps: timestamps, since: since, lines: lines, grep: grep)), type: "Proxy"
+      end
+    end
+  end
+
+  desc "remove", "Remove proxy container and image from servers"
+  option :force, type: :boolean, default: false, desc: "Force removing proxy when apps are still installed"
+  def remove
+    with_lock do
+      if removal_allowed?(options[:force])
+        stop
+        remove_container
+        remove_image
+        remove_proxy_directory
+      end
+    end
+  end
+
+  desc "remove_container", "Remove proxy container from servers", hide: true
+  def remove_container
+    with_lock do
+      on(KAMAL.proxy_hosts) do
+        execute *KAMAL.auditor.record("Removed proxy container"), verbosity: :debug
+        execute *KAMAL.proxy.remove_container
+      end
+    end
+  end
+
+  desc "remove_image", "Remove proxy image from servers", hide: true
+  def remove_image
+    with_lock do
+      on(KAMAL.proxy_hosts) do
+        execute *KAMAL.auditor.record("Removed proxy image"), verbosity: :debug
+        execute *KAMAL.proxy.remove_image
+      end
+    end
+  end
+
+  desc "remove_proxy_directory", "Remove the proxy directory from servers", hide: true
+  def remove_proxy_directory
+    with_lock do
+      on(KAMAL.proxy_hosts) do
+        execute *KAMAL.proxy.remove_proxy_directory, raise_on_non_zero_exit: false
+      end
+    end
+  end
+
+  private
+    def removal_allowed?(force)
+      on(KAMAL.proxy_hosts) do |host|
+        app_count = capture_with_info(*KAMAL.server.app_directory_count).chomp.to_i
+        raise "The are other applications installed on #{host}" if app_count > 0
+      end
+
+      true
+    rescue SSHKit::Runner::ExecuteError => e
+      raise unless e.message.include?("The are other applications installed on")
+
+      if force
+        say "Forcing, so removing the proxy, even though other apps are installed", :magenta
+      else
+        say "Not removing the proxy, as other apps are installed, ignore this check with kamal proxy remove --force", :magenta
+      end
+
+      force
+    end
+end

lib/kamal/cli/prune.rb

@@ -1,7 +1,7 @@
 class Kamal::Cli::Prune < Kamal::Cli::Base
   desc "all", "Prune unused images and stopped containers"
   def all
-    mutating do
+    with_lock do
       containers
       images
     end
@@ -9,7 +9,7 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
 
   desc "images", "Prune unused images"
   def images
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pruned images"), verbosity: :debug
         execute *KAMAL.prune.dangling_images
@@ -24,11 +24,10 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
     retain = options.fetch(:retain, KAMAL.config.retain_containers)
     raise "retain must be at least 1" if retain < 1
 
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug
         execute *KAMAL.prune.app_containers(retain: retain)
-        execute *KAMAL.prune.healthcheck_containers
       end
     end
   end

lib/kamal/cli/registry.rb

@@ -1,18 +1,27 @@
 class Kamal::Cli::Registry < Kamal::Cli::Base
-  desc "login", "Log in to registry locally and remotely"
-  def login
-    run_locally    { execute *KAMAL.registry.login }
-    on(KAMAL.hosts) { execute *KAMAL.registry.login }
-  # FIXME: This rescue needed?
-  rescue ArgumentError => e
-    puts e.message
+  desc "setup", "Setup local registry or log in to remote registry locally and remotely"
+  option :skip_local, aliases: "-L", type: :boolean, default: false, desc: "Skip local login"
+  option :skip_remote, aliases: "-R", type: :boolean, default: false, desc: "Skip remote login"
+  def setup
+    ensure_docker_installed unless options[:skip_local]
+
+    if KAMAL.registry.local?
+      run_locally    { execute *KAMAL.registry.setup } unless options[:skip_local]
+    else
+      run_locally    { execute *KAMAL.registry.login } unless options[:skip_local]
+      on(KAMAL.hosts) { execute *KAMAL.registry.login } unless options[:skip_remote]
+    end
   end
 
-  desc "logout", "Log out of registry remotely"
-  def logout
-    on(KAMAL.hosts) { execute *KAMAL.registry.logout }
-  # FIXME: This rescue needed?
-  rescue ArgumentError => e
-    puts e.message
+  desc "remove", "Remove local registry or log out of remote registry locally and remotely"
+  option :skip_local, aliases: "-L", type: :boolean, default: false, desc: "Skip local login"
+  option :skip_remote, aliases: "-R", type: :boolean, default: false, desc: "Skip remote login"
+  def remove
+    if KAMAL.registry.local?
+      run_locally    { execute *KAMAL.registry.remove, raise_on_non_zero_exit: false } unless options[:skip_local]
+    else
+      run_locally    { execute *KAMAL.registry.logout } unless options[:skip_local]
+      on(KAMAL.hosts) { execute *KAMAL.registry.logout } unless options[:skip_remote]
+    end
   end
 end

lib/kamal/cli/secrets.rb

@@ -0,0 +1,49 @@
+class Kamal::Cli::Secrets < Kamal::Cli::Base
+  desc "fetch [SECRETS...]", "Fetch secrets from a vault"
+  option :adapter, type: :string, aliases: "-a", required: true, desc: "Which vault adapter to use"
+  option :account, type: :string, required: false, desc: "The account identifier or username"
+  option :from, type: :string, required: false, desc: "A vault or folder to fetch the secrets from"
+  option :inline, type: :boolean, required: false, hidden: true
+  def fetch(*secrets)
+    adapter = initialize_adapter(options[:adapter])
+
+    if adapter.requires_account? && options[:account].blank?
+      return puts "No value provided for required options '--account'"
+    end
+
+    results = adapter.fetch(secrets, **options.slice(:account, :from).symbolize_keys)
+
+    return_or_puts JSON.dump(results).shellescape, inline: options[:inline]
+  end
+
+  desc "extract", "Extract a single secret from the results of a fetch call"
+  option :inline, type: :boolean, required: false, hidden: true
+  def extract(name, secrets)
+    parsed_secrets = JSON.parse(secrets)
+    value = parsed_secrets[name] || parsed_secrets.find { |k, v| k.end_with?("/#{name}") }&.last
+
+    raise "Could not find secret #{name}" if value.nil?
+
+    return_or_puts value, inline: options[:inline]
+  end
+
+  desc "print", "Print the secrets (for debugging)"
+  def print
+    KAMAL.config.secrets.to_h.each do |key, value|
+      puts "#{key}=#{value}"
+    end
+  end
+
+  private
+    def initialize_adapter(adapter)
+      Kamal::Secrets::Adapters.lookup(adapter)
+    end
+
+    def return_or_puts(value, inline: nil)
+      if inline
+        value
+      else
+        puts value
+      end
+    end
+end

lib/kamal/cli/server.rb

@@ -1,25 +1,50 @@
 class Kamal::Cli::Server < Kamal::Cli::Base
+  desc "exec", "Run a custom command on the server (use --help to show options)"
+  option :interactive, type: :boolean, aliases: "-i", default: false, desc: "Run the command interactively (use for console/bash)"
+  def exec(*cmd)
+    pre_connect_if_required
+
+    cmd = Kamal::Utils.join_commands(cmd)
+    hosts = KAMAL.hosts
+
+    case
+    when options[:interactive]
+      host = KAMAL.primary_host
+
+      say "Running '#{cmd}' on #{host} interactively...", :magenta
+
+      run_locally { exec KAMAL.server.run_over_ssh(cmd, host: host) }
+    else
+      say "Running '#{cmd}' on #{hosts.join(', ')}...", :magenta
+
+      on(hosts) do |host|
+        execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{host}"), verbosity: :debug
+        puts_by_host host, capture_with_info(cmd)
+      end
+    end
+  end
+
   desc "bootstrap", "Set up Docker to run Kamal apps"
   def bootstrap
-    missing = []
+    with_lock do
+      missing = []
 
-    on(KAMAL.hosts | KAMAL.accessory_hosts) do |host|
-      unless execute(*KAMAL.docker.installed?, raise_on_non_zero_exit: false)
-        if execute(*KAMAL.docker.superuser?, raise_on_non_zero_exit: false)
-          info "Missing Docker on #{host}. Installing…"
-          execute *KAMAL.docker.install
-        else
-          missing << host
+      on(KAMAL.hosts) do |host|
+        unless execute(*KAMAL.docker.installed?, raise_on_non_zero_exit: false)
+          if execute(*KAMAL.docker.superuser?, raise_on_non_zero_exit: false)
+            info "Missing Docker on #{host}. Installing…"
+            execute *KAMAL.docker.install
+          else
+            missing << host
+          end
         end
       end
 
-      execute(*KAMAL.server.ensure_run_directory)
-    end
+      if missing.any?
+        raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and either `wget` or `curl`. Install Docker manually: https://docs.docker.com/engine/install/"
+      end
 
-    if missing.any?
-      raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and either `wget` or `curl`. Install Docker manually: https://docs.docker.com/engine/install/"
+      run_hook "docker-setup"
     end
-
-    run_hook "docker-setup"
   end
 end

lib/kamal/cli/templates/deploy.yml

@@ -2,45 +2,82 @@
 service: my-app
 
 # Name of the container image.
-image: user/my-app
+image: my-user/my-app
 
 # Deploy to these servers.
 servers:
-  - 192.168.0.1
+  web:
+    - 192.168.0.1
+  # job:
+  #   hosts:
+  #     - 192.168.0.1
+  #   cmd: bin/jobs
+
+# Enable SSL auto certification via Let's Encrypt and allow for multiple apps on a single web server.
+# Remove this section when using multiple web servers and ensure you terminate SSL at your load balancer.
+#
+# Note: If using Cloudflare, set encryption mode in SSL/TLS setting to "Full" to enable CF-to-app encryption.
+proxy:
+  ssl: true
+  host: app.example.com
+  # Proxy connects to your container on port 80 by default.
+  # app_port: 3000
 
 # Credentials for your image host.
 registry:
+  server: localhost:5555
   # Specify the registry server, if you're not using Docker Hub
   # server: registry.digitalocean.com / ghcr.io / ...
-  username: my-user
+  # username: my-user
 
-  # Always use an access token rather than real password when possible.
-  password:
-    - KAMAL_REGISTRY_PASSWORD
+  # Always use an access token rather than real password (pulled from .kamal/secrets).
+  # password:
+  #   - KAMAL_REGISTRY_PASSWORD
 
-# Inject ENV variables into containers (secrets come from .env).
-# Remember to run `kamal env push` after making changes!
+# Configure builder setup.
+builder:
+  arch: amd64
+  # Pass in additional build args needed for your Dockerfile.
+  # args:
+  #   RUBY_VERSION: <%= ENV["RBENV_VERSION"] || ENV["rvm_ruby_string"] || "#{RUBY_ENGINE}-#{RUBY_ENGINE_VERSION}" %>
+# Inject ENV variables into containers (secrets come from .kamal/secrets).
+#
 # env:
 #   clear:
 #     DB_HOST: 192.168.0.2
 #   secret:
 #     - RAILS_MASTER_KEY
 
+# Aliases are triggered with "bin/kamal <alias>". You can overwrite arguments on invocation:
+# "bin/kamal app logs -r job" will tail logs from the first server in the job section.
+#
+# aliases:
+#   shell: app exec --interactive --reuse "bash"
+
 # Use a different ssh user than root
+#
 # ssh:
 #   user: app
 
-# Configure builder setup.
-# builder:
-#   args:
-#     RUBY_VERSION: 3.2.0
-#   secrets:
-#     - GITHUB_TOKEN
-#   remote:
-#     arch: amd64
-#     host: ssh://app@192.168.0.1
+# Use a persistent storage volume.
+#
+# volumes:
+#   - "app_storage:/app/storage"
 
-# Use accessory services (secrets come from .env).
+# Bridge fingerprinted assets, like JS and CSS, between versions to avoid
+# hitting 404 on in-flight requests. Combines all files from new and old
+# version inside the asset_path.
+#
+# asset_path: /app/public/assets
+
+# Configure rolling deploys by setting a wait time between batches of restarts.
+#
+# boot:
+#   limit: 10 # Can also specify as a percentage of total hosts, such as "25%"
+#   wait: 2
+
+# Use accessory services (secrets come from .kamal/secrets).
+#
 # accessories:
 #   db:
 #     image: mysql:8.0
@@ -57,45 +94,8 @@ registry:
 #     directories:
 #       - data:/var/lib/mysql
 #   redis:
-#     image: redis:7.0
+#     image: valkey/valkey:8
 #     host: 192.168.0.2
 #     port: 6379
 #     directories:
 #       - data:/data
-
-# Configure custom arguments for Traefik. Be sure to reboot traefik when you modify it.
-# traefik:
-#   args:
-#     accesslog: true
-#     accesslog.format: json
-
-# Configure a custom healthcheck (default is /up on port 3000)
-# healthcheck:
-#   path: /healthz
-#   port: 4000
-
-# Bridge fingerprinted assets, like JS and CSS, between versions to avoid
-# hitting 404 on in-flight requests. Combines all files from new and old
-# version inside the asset_path.
-#
-# If your app is using the Sprockets gem, ensure it sets `config.assets.manifest`.
-# See https://github.com/basecamp/kamal/issues/626 for details
-#
-# asset_path: /rails/public/assets
-
-# Configure rolling deploys by setting a wait time between batches of restarts.
-# boot:
-#   limit: 10 # Can also specify as a percentage of total hosts, such as "25%"
-#   wait: 2
-
-# Configure the role used to determine the primary_host. This host takes
-# deploy locks, runs health checks during the deploy, and follow logs, etc.
-#
-# Caution: there's no support for role renaming yet, so be careful to cleanup
-#          the previous role on the deployed hosts.
-# primary_role: web
-
-# Controls if we abort when see a role with no hosts. Disabling this may be
-# useful for more complex deploy configurations.
-#
-# allow_empty_roles: false

lib/kamal/cli/templates/sample_hooks/docker-setup.sample

@@ -1,7 +1,3 @@
-#!/usr/bin/env ruby
+#!/bin/sh
 
-# A sample docker-setup hook
-#
-# Sets up a Docker network which can then be used by the application’s containers
-
-ssh user@example.com docker network create kamal
+echo "Docker set up on $KAMAL_HOSTS..."

lib/kamal/cli/templates/sample_hooks/post-app-boot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Booted app version $KAMAL_VERSION on $KAMAL_HOSTS..."

lib/kamal/cli/templates/sample_hooks/post-deploy.sample

@@ -7,7 +7,7 @@
 # KAMAL_PERFORMER
 # KAMAL_VERSION
 # KAMAL_HOSTS
-# KAMAL_ROLE (if set)
+# KAMAL_ROLES (if set)
 # KAMAL_DESTINATION (if set)
 # KAMAL_RUNTIME
 

lib/kamal/cli/templates/sample_hooks/post-proxy-reboot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooted kamal-proxy on $KAMAL_HOSTS"

lib/kamal/cli/templates/sample_hooks/post-traefik-reboot.sample

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-echo "Rebooted Traefik on $KAMAL_HOSTS"

lib/kamal/cli/templates/sample_hooks/pre-app-boot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Booting app version $KAMAL_VERSION on $KAMAL_HOSTS..."

lib/kamal/cli/templates/sample_hooks/pre-build.sample

@@ -13,7 +13,7 @@
 # KAMAL_PERFORMER
 # KAMAL_VERSION
 # KAMAL_HOSTS
-# KAMAL_ROLE (if set)
+# KAMAL_ROLES (if set)
 # KAMAL_DESTINATION (if set)
 
 if [ -n "$(git status --porcelain)" ]; then

lib/kamal/cli/templates/sample_hooks/pre-connect.sample

@@ -9,7 +9,7 @@
 # KAMAL_PERFORMER
 # KAMAL_VERSION
 # KAMAL_HOSTS
-# KAMAL_ROLE (if set)
+# KAMAL_ROLES (if set)
 # KAMAL_DESTINATION (if set)
 # KAMAL_RUNTIME
 

lib/kamal/cli/templates/sample_hooks/pre-deploy.sample

@@ -13,7 +13,7 @@
 # KAMAL_HOSTS
 # KAMAL_COMMAND
 # KAMAL_SUBCOMMAND
-# KAMAL_ROLE (if set)
+# KAMAL_ROLES (if set)
 # KAMAL_DESTINATION (if set)
 
 # Only check the build status for production deployments
@@ -43,7 +43,7 @@ class GithubStatusChecks
   attr_reader :remote_url, :git_sha, :github_client, :combined_status
 
   def initialize
-    @remote_url = `git config --get remote.origin.url`.strip.delete_prefix("https://github.com/")
+    @remote_url = github_repo_from_remote_url
     @git_sha = `git rev-parse HEAD`.strip
     @github_client = Octokit::Client.new(access_token: ENV["GITHUB_TOKEN"])
     refresh!
@@ -77,16 +77,29 @@ class GithubStatusChecks
       "Build not started..."
     end
   end
+
+  private
+    def github_repo_from_remote_url
+      url = `git config --get remote.origin.url`.strip.delete_suffix(".git")
+      if url.start_with?("https://github.com/")
+        url.delete_prefix("https://github.com/")
+      elsif url.start_with?("git@github.com:")
+        url.delete_prefix("git@github.com:")
+      else
+        url
+      end
+    end
 end
 
 
 $stdout.sync = true
 
-puts "Checking build status..."
-attempts = 0
-checks = GithubStatusChecks.new
-
 begin
+  puts "Checking build status..."
+
+  attempts = 0
+  checks = GithubStatusChecks.new
+
   loop do
     case checks.state
     when "success"

lib/kamal/cli/templates/sample_hooks/pre-proxy-reboot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooting kamal-proxy on $KAMAL_HOSTS..."

lib/kamal/cli/templates/sample_hooks/pre-traefik-reboot.sample

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-echo "Rebooting Traefik on $KAMAL_HOSTS..."

lib/kamal/cli/templates/secrets

@@ -0,0 +1,17 @@
+# Secrets defined here are available for reference under registry/password, env/secret, builder/secrets,
+# and accessories/*/env/secret in config/deploy.yml. All secrets should be pulled from either
+# password manager, ENV, or a file. DO NOT ENTER RAW CREDENTIALS HERE! This file needs to be safe for git.
+
+# Option 1: Read secrets from the environment
+# KAMAL_REGISTRY_PASSWORD=$KAMAL_REGISTRY_PASSWORD
+
+# Option 2: Read secrets via a command
+# RAILS_MASTER_KEY=$(cat config/master.key)
+
+# Option 3: Read secrets via kamal secrets helpers
+# These will handle logging in and fetching the secrets in as few calls as possible
+# There are adapters for 1Password, LastPass + Bitwarden
+#
+# SECRETS=$(kamal secrets fetch --adapter 1password --account my-account --from MyVault/MyItem KAMAL_REGISTRY_PASSWORD RAILS_MASTER_KEY)
+# KAMAL_REGISTRY_PASSWORD=$(kamal secrets extract KAMAL_REGISTRY_PASSWORD $SECRETS)
+# RAILS_MASTER_KEY=$(kamal secrets extract RAILS_MASTER_KEY $SECRETS)

lib/kamal/cli/templates/template.env

@@ -1,2 +0,0 @@
-KAMAL_REGISTRY_PASSWORD=change-this
-RAILS_MASTER_KEY=another-env

lib/kamal/cli/traefik.rb

@@ -1,120 +0,0 @@
-class Kamal::Cli::Traefik < Kamal::Cli::Base
-  desc "boot", "Boot Traefik on servers"
-  def boot
-    mutating do
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.registry.login
-        execute *KAMAL.traefik.start_or_run
-      end
-    end
-  end
-
-  desc "reboot", "Reboot Traefik on servers (stop container, remove container, start new container)"
-  option :rolling, type: :boolean, default: false, desc: "Reboot traefik on hosts in sequence, rather than in parallel"
-  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
-  def reboot
-    confirming "This will cause a brief outage on each host. Are you sure?" do
-      mutating do
-        host_groups = options[:rolling] ? KAMAL.traefik_hosts : [ KAMAL.traefik_hosts ]
-        host_groups.each do |hosts|
-          host_list = Array(hosts).join(",")
-          run_hook "pre-traefik-reboot", hosts: host_list
-          on(hosts) do
-            execute *KAMAL.auditor.record("Rebooted traefik"), verbosity: :debug
-            execute *KAMAL.registry.login
-            execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
-            execute *KAMAL.traefik.remove_container
-            execute *KAMAL.traefik.run
-          end
-          run_hook "post-traefik-reboot", hosts: host_list
-        end
-      end
-    end
-  end
-
-  desc "start", "Start existing Traefik container on servers"
-  def start
-    mutating do
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.auditor.record("Started traefik"), verbosity: :debug
-        execute *KAMAL.traefik.start
-      end
-    end
-  end
-
-  desc "stop", "Stop existing Traefik container on servers"
-  def stop
-    mutating do
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.auditor.record("Stopped traefik"), verbosity: :debug
-        execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
-      end
-    end
-  end
-
-  desc "restart", "Restart existing Traefik container on servers"
-  def restart
-    mutating do
-      stop
-      start
-    end
-  end
-
-  desc "details", "Show details about Traefik container from servers"
-  def details
-    on(KAMAL.traefik_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.traefik.info), type: "Traefik" }
-  end
-
-  desc "logs", "Show log lines from Traefik on servers"
-  option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
-  option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server"
-  option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
-  option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
-  def logs
-    grep = options[:grep]
-
-    if options[:follow]
-      run_locally do
-        info "Following logs on #{KAMAL.primary_host}..."
-        info KAMAL.traefik.follow_logs(host: KAMAL.primary_host, grep: grep)
-        exec KAMAL.traefik.follow_logs(host: KAMAL.primary_host, grep: grep)
-      end
-    else
-      since = options[:since]
-      lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
-
-      on(KAMAL.traefik_hosts) do |host|
-        puts_by_host host, capture(*KAMAL.traefik.logs(since: since, lines: lines, grep: grep)), type: "Traefik"
-      end
-    end
-  end
-
-  desc "remove", "Remove Traefik container and image from servers"
-  def remove
-    mutating do
-      stop
-      remove_container
-      remove_image
-    end
-  end
-
-  desc "remove_container", "Remove Traefik container from servers", hide: true
-  def remove_container
-    mutating do
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.auditor.record("Removed traefik container"), verbosity: :debug
-        execute *KAMAL.traefik.remove_container
-      end
-    end
-  end
-
-  desc "remove_image", "Remove Traefik image from servers", hide: true
-  def remove_image
-    mutating do
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.auditor.record("Removed traefik image"), verbosity: :debug
-        execute *KAMAL.traefik.remove_image
-      end
-    end
-  end
-end

lib/kamal/commander.rb

@@ -1,17 +1,27 @@
 require "active_support/core_ext/enumerable"
 require "active_support/core_ext/module/delegation"
+require "active_support/core_ext/object/blank"
 
 class Kamal::Commander
-  attr_accessor :verbosity, :holding_lock, :hold_lock_on_error
+  attr_accessor :verbosity, :holding_lock, :connected
+  attr_reader :specific_roles, :specific_hosts
+  delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :app_hosts, :proxy_hosts, :accessory_hosts, to: :specifics
 
   def initialize
+    reset
+  end
+
+  def reset
     self.verbosity = :info
-    self.holding_lock = false
-    self.hold_lock_on_error = false
+    self.holding_lock = ENV["KAMAL_LOCK"] == "true"
+    self.connected = false
+    @specifics = @specific_roles = @specific_hosts = nil
+    @config = @config_kwargs = nil
+    @commands = {}
   end
 
   def config
-    @config ||= Kamal::Configuration.create_from(**@config_kwargs).tap do |config|
+    @config ||= Kamal::Configuration.create_from(**@config_kwargs.to_h).tap do |config|
       @config_kwargs = nil
       configure_sshkit_with(config)
     end
@@ -21,13 +31,21 @@ class Kamal::Commander
     @config, @config_kwargs = nil, kwargs
   end
 
-  attr_reader :specific_roles, :specific_hosts
+  def configured?
+    @config || @config_kwargs
+  end
 
   def specific_primary!
-    self.specific_hosts = [ config.primary_host ]
+    @specifics = nil
+    if specific_roles.present?
+      self.specific_hosts = [ specific_roles.first.primary_host ]
+    else
+      self.specific_hosts = [ config.primary_host ]
+    end
   end
 
   def specific_roles=(role_names)
+    @specifics = nil
     if role_names.present?
       @specific_roles = Kamal::Utils.filter_specific_items(role_names, config.roles)
 
@@ -40,6 +58,7 @@ class Kamal::Commander
   end
 
   def specific_hosts=(hosts)
+    @specifics = nil
     if hosts.present?
       @specific_hosts = Kamal::Utils.filter_specific_items(hosts, config.all_hosts)
 
@@ -51,50 +70,19 @@ class Kamal::Commander
     end
   end
 
-  def primary_host
-    # Given a list of specific roles, make an effort to match up with the primary_role
-    specific_hosts&.first || specific_roles&.detect { |role| role == config.primary_role }&.primary_host || specific_roles&.first&.primary_host || config.primary_host
-  end
-
-  def primary_role
-    roles_on(primary_host).first
-  end
-
-  def roles
-    (specific_roles || config.roles).select do |role|
-      ((specific_hosts || config.all_hosts) & role.hosts).any?
-    end
-  end
-
-  def hosts
-    (specific_hosts || config.all_hosts).select do |host|
-      (specific_roles || config.roles).flat_map(&:hosts).include?(host)
-    end
-  end
-
-  def roles_on(host)
-    roles.select { |role| role.hosts.include?(host.to_s) }
-  end
-
-  def traefik_hosts
-    specific_hosts || config.traefik_hosts
-  end
-
-  def accessory_hosts
-    specific_hosts || config.accessories.flat_map(&:hosts)
+  def with_specific_hosts(hosts)
+    original_hosts, self.specific_hosts = specific_hosts, hosts
+    yield
+  ensure
+    self.specific_hosts = original_hosts
   end
 
   def accessory_names
     config.accessories&.collect(&:name) || []
   end
 
-  def accessories_on(host)
-    config.accessories.select { |accessory| accessory.hosts.include?(host.to_s) }.map(&:name)
-  end
-
-
-  def app(role: nil)
-    Kamal::Commands::App.new(config, role: role)
+  def app(role: nil, host: nil)
+    Kamal::Commands::App.new(config, role: role, host: host)
   end
 
   def accessory(name)
@@ -106,42 +94,41 @@ class Kamal::Commander
   end
 
   def builder
-    @builder ||= Kamal::Commands::Builder.new(config)
+    @commands[:builder] ||= Kamal::Commands::Builder.new(config)
   end
 
   def docker
-    @docker ||= Kamal::Commands::Docker.new(config)
-  end
-
-  def healthcheck
-    @healthcheck ||= Kamal::Commands::Healthcheck.new(config)
+    @commands[:docker] ||= Kamal::Commands::Docker.new(config)
   end
 
   def hook
-    @hook ||= Kamal::Commands::Hook.new(config)
+    @commands[:hook] ||= Kamal::Commands::Hook.new(config)
   end
 
   def lock
-    @lock ||= Kamal::Commands::Lock.new(config)
+    @commands[:lock] ||= Kamal::Commands::Lock.new(config)
+  end
+
+  def proxy
+    @commands[:proxy] ||= Kamal::Commands::Proxy.new(config)
   end
 
   def prune
-    @prune ||= Kamal::Commands::Prune.new(config)
+    @commands[:prune] ||= Kamal::Commands::Prune.new(config)
   end
 
   def registry
-    @registry ||= Kamal::Commands::Registry.new(config)
+    @commands[:registry] ||= Kamal::Commands::Registry.new(config)
   end
 
   def server
-    @server ||= Kamal::Commands::Server.new(config)
+    @commands[:server] ||= Kamal::Commands::Server.new(config)
   end
 
-  def traefik
-    @traefik ||= Kamal::Commands::Traefik.new(config)
+  def alias(name)
+    config.aliases[name]
   end
 
-
   def with_verbosity(level)
     old_level = self.verbosity
 
@@ -154,20 +141,12 @@ class Kamal::Commander
     SSHKit.config.output_verbosity = old_level
   end
 
-  def boot_strategy
-    if config.boot.limit.present?
-      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
-    else
-      {}
-    end
-  end
-
   def holding_lock?
     self.holding_lock
   end
 
-  def hold_lock_on_error?
-    self.hold_lock_on_error
+  def connected?
+    self.connected
   end
 
   private
@@ -181,4 +160,8 @@ class Kamal::Commander
       SSHKit.config.command_map[:docker] = "docker" # No need to use /usr/bin/env, just clogs up the logs
       SSHKit.config.output_verbosity = verbosity
     end
+
+    def specifics
+      @specifics ||= Kamal::Commander::Specifics.new(config, specific_hosts, specific_roles)
+    end
 end

lib/kamal/commander/specifics.rb

@@ -0,0 +1,62 @@
+class Kamal::Commander::Specifics
+  attr_reader :primary_host, :primary_role, :hosts, :roles
+  delegate :stable_sort!, to: Kamal::Utils
+
+  def initialize(config, specific_hosts, specific_roles)
+    @config, @specific_hosts, @specific_roles = config, specific_hosts, specific_roles
+
+    @roles, @hosts = specified_roles, specified_hosts
+
+    @primary_host = specific_hosts&.first || primary_specific_role&.primary_host || config.primary_host
+    @primary_role = primary_or_first_role(roles_on(primary_host))
+
+    stable_sort!(roles) { |role| role == primary_role ? 0 : 1 }
+    sort_primary_role_hosts_first!(hosts)
+  end
+
+  def roles_on(host)
+    roles.select { |role| role.hosts.include?(host.to_s) }
+  end
+
+  def app_hosts
+    @app_hosts ||= sort_primary_role_hosts_first!(config.app_hosts & specified_hosts)
+  end
+
+  def proxy_hosts
+    config.proxy_hosts & specified_hosts
+  end
+
+  def accessory_hosts
+    config.accessories.flat_map(&:hosts) & specified_hosts
+  end
+
+  private
+    attr_reader :config, :specific_hosts, :specific_roles
+
+    def primary_specific_role
+      primary_or_first_role(specific_roles) if specific_roles.present?
+    end
+
+    def primary_or_first_role(roles)
+      roles.detect { |role| role == config.primary_role } || roles.first
+    end
+
+    def specified_roles
+      (specific_roles || config.roles) \
+        .select { |role| ((specific_hosts || config.all_hosts) & role.hosts).any? }
+    end
+
+    def specified_hosts
+      specified_hosts = specific_hosts || config.all_hosts
+
+      if (specific_role_hosts = specific_roles&.flat_map(&:hosts)).present?
+        specified_hosts.select { |host| specific_role_hosts.include?(host) }
+      else
+        specified_hosts
+      end
+    end
+
+    def sort_primary_role_hosts_first!(hosts)
+      stable_sort!(hosts) { |host| roles_on(host).any? { |role| role == primary_role } ? 0 : 1 }
+    end
+end

lib/kamal/commands/accessory.rb

@@ -1,20 +1,26 @@
 class Kamal::Commands::Accessory < Kamal::Commands::Base
+  include Proxy
+
   attr_reader :accessory_config
   delegate :service_name, :image, :hosts, :port, :files, :directories, :cmd,
-           :publish_args, :env_args, :volume_args, :label_args, :option_args, to: :accessory_config
+           :network_args, :publish_args, :env_args, :volume_args, :label_args, :option_args,
+           :secrets_io, :secrets_path, :env_directory, :proxy, :running_proxy?, :registry,
+           to: :accessory_config
 
   def initialize(config, name:)
     super(config)
     @accessory_config = config.accessory(name)
   end
 
-  def run
+  def run(host: nil)
     docker :run,
       "--name", service_name,
       "--detach",
       "--restart", "unless-stopped",
+      *network_args,
       *config.logging_args,
       *publish_args,
+      *([ "--env", "KAMAL_HOST=\"#{host}\"" ] if host),
       *env_args,
       *volume_args,
       *label_args,
@@ -31,36 +37,35 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     docker :container, :stop, service_name
   end
 
-  def info
-    docker :ps, *service_filter
+  def info(all: false, quiet: false)
+    docker :ps, *("-a" if all), *("-q" if quiet), *service_filter
   end
 
-
-  def logs(since: nil, lines: nil, grep: nil)
+  def logs(timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil)
     pipe \
-      docker(:logs, service_name, (" --since #{since}" if since), (" --tail #{lines}" if lines), "--timestamps", "2>&1"),
-      ("grep '#{grep}'" if grep)
+      docker(:logs, service_name, (" --since #{since}" if since), (" --tail #{lines}" if lines), ("--timestamps" if timestamps), "2>&1"),
+      ("grep '#{grep}'#{" #{grep_options}" if grep_options}" if grep)
   end
 
-  def follow_logs(grep: nil)
+  def follow_logs(timestamps: true, grep: nil, grep_options: nil)
     run_over_ssh \
       pipe \
-        docker(:logs, service_name, "--timestamps", "--tail", "10", "--follow", "2>&1"),
-        (%(grep "#{grep}") if grep)
+        docker(:logs, service_name, ("--timestamps" if timestamps), "--tail", "10", "--follow", "2>&1"),
+        (%(grep "#{grep}"#{" #{grep_options}" if grep_options}) if grep)
   end
 
-
   def execute_in_existing_container(*command, interactive: false)
     docker :exec,
-      ("-it" if interactive),
+      (docker_interactive_args if interactive),
       service_name,
       *command
   end
 
   def execute_in_new_container(*command, interactive: false)
     docker :run,
-      ("-it" if interactive),
+      (docker_interactive_args if interactive),
       "--rm",
+      *network_args,
       *env_args,
       *volume_args,
       image,
@@ -79,7 +84,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     super command, host: hosts.first
   end
 
-
   def ensure_local_file_present(local_file)
     if !local_file.is_a?(StringIO) && !Pathname.new(local_file).exist?
       raise "Missing file: #{local_file}"
@@ -98,12 +102,8 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     docker :image, :rm, "--force", image
   end
 
-  def make_env_directory
-    make_directory accessory_config.env.secrets_directory
-  end
-
-  def remove_env_file
-    [ :rm, "-f", accessory_config.env.secrets_file ]
+  def ensure_env_directory
+    make_directory env_directory
   end
 
   private

lib/kamal/commands/accessory/proxy.rb

@@ -0,0 +1,16 @@
+module Kamal::Commands::Accessory::Proxy
+  delegate :container_name, to: :"config.proxy_boot", prefix: :proxy
+
+  def deploy(target:)
+    proxy_exec :deploy, service_name, *proxy.deploy_command_args(target: target)
+  end
+
+  def remove
+    proxy_exec :remove, service_name
+  end
+
+  private
+    def proxy_exec(*command)
+      docker :exec, proxy_container_name, "kamal-proxy", *command
+    end
+end

lib/kamal/commands/app.rb

@@ -1,13 +1,16 @@
 class Kamal::Commands::App < Kamal::Commands::Base
-  include Assets, Containers, Cord, Execution, Images, Logging
+  include Assets, Containers, ErrorPages, Execution, Images, Logging, Proxy
 
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 
-  attr_reader :role, :role
+  attr_reader :role, :host
 
-  def initialize(config, role: nil)
+  delegate :container_name, to: :role
+
+  def initialize(config, role: nil, host: nil)
     super(config)
     @role = role
+    @host = host
   end
 
   def run(hostname: nil)
@@ -15,11 +18,12 @@ class Kamal::Commands::App < Kamal::Commands::Base
       "--detach",
       "--restart unless-stopped",
       "--name", container_name,
+      "--network", "kamal",
       *([ "--hostname", hostname ] if hostname),
-      "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
-      "-e", "KAMAL_VERSION=\"#{config.version}\"",
-      *role.env_args,
-      *role.health_check_args,
+      "--env", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
+      "--env", "KAMAL_VERSION=\"#{config.version}\"",
+      "--env", "KAMAL_HOST=\"#{host}\"",
+      *role.env_args(host),
       *role.logging_args,
       *config.volume_args,
       *role.asset_volume_args,
@@ -40,16 +44,16 @@ class Kamal::Commands::App < Kamal::Commands::Base
   def stop(version: nil)
     pipe \
       version ? container_id_for_version(version) : current_running_container_id,
-      xargs(config.stop_wait_time ? docker(:stop, "-t", config.stop_wait_time) : docker(:stop))
+      xargs(docker(:stop, *role.stop_args))
   end
 
   def info
-    docker :ps, *filter_args
+    docker :ps, *container_filter_args
   end
 
 
   def current_running_container_id
-    docker :ps, "--quiet", *filter_args(statuses: ACTIVE_DOCKER_STATUSES), "--latest"
+    current_running_container(format: "--quiet")
   end
 
   def container_id_for_version(version, only_running: false)
@@ -57,41 +61,64 @@ class Kamal::Commands::App < Kamal::Commands::Base
   end
 
   def current_running_version
-    list_versions("--latest", statuses: ACTIVE_DOCKER_STATUSES)
+    pipe \
+      current_running_container(format: "--format '{{.Names}}'"),
+      extract_version_from_name
   end
 
   def list_versions(*docker_args, statuses: nil)
     pipe \
-      docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
-      %(while read line; do echo ${line##{role.container_prefix}-}; done) # Extract SHA from "service-role-dest-SHA"
+      docker(:ps, *container_filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
+      extract_version_from_name
   end
 
-
-  def make_env_directory
-    make_directory role.env.secrets_directory
+  def ensure_env_directory
+    make_directory role.env_directory
   end
 
-  def remove_env_file
-    [ :rm, "-f", role.env.secrets_file ]
-  end
-
-
   private
-    def container_name(version = nil)
-      [ role.container_prefix, version || config.version ].compact.join("-")
+    def latest_image_id
+      docker :image, :ls, *argumentize("--filter", "reference=#{config.latest_image}"), "--format", "'{{.ID}}'"
     end
 
-    def filter_args(statuses: nil)
-      argumentize "--filter", filters(statuses: statuses)
+    def current_running_container(format:)
+      pipe \
+        shell(chain(latest_image_container(format: format), latest_container(format: format))),
+        [ :head, "-1" ]
     end
 
-    def filters(statuses: nil)
+    def latest_image_container(format:)
+      latest_container format: format, filters: [ "ancestor=$(#{latest_image_id.join(" ")})" ]
+    end
+
+    def latest_container(format:, filters: nil)
+      docker :ps, "--latest", *format, *container_filter_args(statuses: ACTIVE_DOCKER_STATUSES), argumentize("--filter", filters)
+    end
+
+    def container_filter_args(statuses: nil)
+      argumentize "--filter", container_filters(statuses: statuses)
+    end
+
+    def image_filter_args
+      argumentize "--filter", image_filters
+    end
+
+    def extract_version_from_name
+      # Extract SHA from "service-role-dest-SHA"
+      %(while read line; do echo ${line##{role.container_prefix}-}; done)
+    end
+
+    def container_filters(statuses: nil)
       [ "label=service=#{config.service}" ].tap do |filters|
-        filters << "label=destination=#{config.destination}" if config.destination
+        filters << "label=destination=#{config.destination}"
         filters << "label=role=#{role}" if role
         statuses&.each do |status|
           filters << "status=#{status}"
         end
       end
     end
+
+    def image_filters
+      [ "label=service=#{config.service}" ]
+    end
 end

lib/kamal/commands/app/assets.rb

@@ -3,18 +3,18 @@ module Kamal::Commands::App::Assets
     asset_container = "#{role.container_prefix}-assets"
 
     combine \
-      make_directory(role.asset_extracted_path),
-      [ *docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true" ],
-      docker(:run, "--name", asset_container, "--detach", "--rm", config.latest_image, "sleep 1000000"),
-      docker(:cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_path),
-      docker(:stop, "-t 1", asset_container),
+      make_directory(role.asset_extracted_directory),
+      [ *docker(:container, :rm, asset_container, "2> /dev/null"), "|| true" ],
+      docker(:container, :create, "--name", asset_container, config.absolute_image),
+      docker(:container, :cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_directory),
+      docker(:container, :rm, asset_container),
       by: "&&"
   end
 
   def sync_asset_volumes(old_version: nil)
-    new_extracted_path, new_volume_path = role.asset_extracted_path(config.version), role.asset_volume.host_path
+    new_extracted_path, new_volume_path = role.asset_extracted_directory(config.version), role.asset_volume.host_path
     if old_version.present?
-      old_extracted_path, old_volume_path = role.asset_extracted_path(old_version), role.asset_volume(old_version).host_path
+      old_extracted_path, old_volume_path = role.asset_extracted_directory(old_version), role.asset_volume(old_version).host_path
     end
 
     commands = [ make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path) ]
@@ -29,8 +29,8 @@ module Kamal::Commands::App::Assets
 
   def clean_up_assets
     chain \
-      find_and_remove_older_siblings(role.asset_extracted_path),
-      find_and_remove_older_siblings(role.asset_volume_path)
+      find_and_remove_older_siblings(role.asset_extracted_directory),
+      find_and_remove_older_siblings(role.asset_volume_directory)
   end
 
   private
@@ -39,7 +39,7 @@ module Kamal::Commands::App::Assets
         :find,
         Pathname.new(path).dirname.to_s,
         "-maxdepth 1",
-        "-name", "'#{role.container_prefix}-*'",
+        "-name", "'#{role.name}-*'",
         "!", "-name", Pathname.new(path).basename.to_s,
         "-exec rm -rf \"{}\" +"
       ]

lib/kamal/commands/app/containers.rb

@@ -1,6 +1,8 @@
 module Kamal::Commands::App::Containers
+  DOCKER_HEALTH_LOG_FORMAT    = "'{{json .State.Health}}'"
+
   def list_containers
-    docker :container, :ls, "--all", *filter_args
+    docker :container, :ls, "--all", *container_filter_args
   end
 
   def list_container_names
@@ -18,6 +20,12 @@ module Kamal::Commands::App::Containers
   end
 
   def remove_containers
-    docker :container, :prune, "--force", *filter_args
+    docker :container, :prune, "--force", *container_filter_args
+  end
+
+  def container_health_log(version:)
+    pipe \
+      container_id_for(container_name: container_name(version)),
+      xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT))
   end
 end

lib/kamal/commands/app/cord.rb

@@ -1,22 +0,0 @@
-module Kamal::Commands::App::Cord
-  def cord(version:)
-    pipe \
-      docker(:inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", container_name(version)),
-      [ :awk, "'$2 == \"#{role.cord_volume.container_path}\" {print $1}'" ]
-  end
-
-  def tie_cord(cord)
-    create_empty_file(cord)
-  end
-
-  def cut_cord(cord)
-    remove_directory(cord)
-  end
-
-  private
-    def create_empty_file(file)
-      chain \
-        make_directory_for(file),
-        [ :touch, file ]
-    end
-end

lib/kamal/commands/app/error_pages.rb

@@ -0,0 +1,9 @@
+module Kamal::Commands::App::ErrorPages
+  def create_error_pages_directory
+    make_directory(config.proxy_boot.error_pages_directory)
+  end
+
+  def clean_up_error_pages
+    [ :find, config.proxy_boot.error_pages_directory, "-mindepth", "1", "-maxdepth", "1", "!", "-name", KAMAL.config.version, "-exec", "rm", "-rf", "{} +" ]
+  end
+end

lib/kamal/commands/app/execution.rb

@@ -1,27 +1,32 @@
 module Kamal::Commands::App::Execution
-  def execute_in_existing_container(*command, interactive: false)
+  def execute_in_existing_container(*command, interactive: false, env:)
     docker :exec,
-      ("-it" if interactive),
+      (docker_interactive_args if interactive),
+      *argumentize("--env", env),
       container_name,
       *command
   end
 
-  def execute_in_new_container(*command, interactive: false)
+  def execute_in_new_container(*command, interactive: false, detach: false, env:)
     docker :run,
-      ("-it" if interactive),
-      "--rm",
-      *role&.env_args,
+      (docker_interactive_args if interactive),
+      ("--detach" if detach),
+      ("--rm" unless detach),
+      "--network", "kamal",
+      *role&.env_args(host),
+      *argumentize("--env", env),
+      *role.logging_args,
       *config.volume_args,
       *role&.option_args,
       config.absolute_image,
       *command
   end
 
-  def execute_in_existing_container_over_ssh(*command, host:)
-    run_over_ssh execute_in_existing_container(*command, interactive: true), host: host
+  def execute_in_existing_container_over_ssh(*command,  env:)
+    run_over_ssh execute_in_existing_container(*command, interactive: true, env: env), host: host
   end
 
-  def execute_in_new_container_over_ssh(*command, host:)
-    run_over_ssh execute_in_new_container(*command, interactive: true), host: host
+  def execute_in_new_container_over_ssh(*command, env:)
+    run_over_ssh execute_in_new_container(*command, interactive: true, env: env), host: host
   end
 end

lib/kamal/commands/app/images.rb

@@ -4,10 +4,10 @@ module Kamal::Commands::App::Images
   end
 
   def remove_images
-    docker :image, :prune, "--all", "--force", *filter_args
+    docker :image, :prune, "--all", "--force", *image_filter_args
   end
 
-  def tag_current_image_as_latest
+  def tag_latest_image
     docker :tag, config.absolute_image, config.latest_image
   end
 end

lib/kamal/commands/app/logging.rb

@@ -1,18 +1,28 @@
 module Kamal::Commands::App::Logging
-  def logs(since: nil, lines: nil, grep: nil)
+  def logs(container_id: nil, timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil)
     pipe \
-      current_running_container_id,
-      "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
-      ("grep '#{grep}'" if grep)
+      container_id_command(container_id),
+      "xargs docker logs#{" --timestamps" if timestamps}#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
+      ("grep '#{grep}'#{" #{grep_options}" if grep_options}" if grep)
   end
 
-  def follow_logs(host:, lines: nil, grep: nil)
+  def follow_logs(host:, container_id: nil, timestamps: true, lines: nil, grep: nil, grep_options: nil)
     run_over_ssh \
       pipe(
-        current_running_container_id,
-        "xargs docker logs --timestamps#{" --tail #{lines}" if lines} --follow 2>&1",
-        (%(grep "#{grep}") if grep)
+        container_id_command(container_id),
+        "xargs docker logs#{" --timestamps" if timestamps}#{" --tail #{lines}" if lines} --follow 2>&1",
+        (%(grep "#{grep}"#{" #{grep_options}" if grep_options}) if grep)
       ),
       host: host
   end
+
+  private
+
+  def container_id_command(container_id)
+    case container_id
+    when Array then container_id
+    when String, Symbol then "echo #{container_id}"
+    else current_running_container_id
+    end
+  end
 end

lib/kamal/commands/app/proxy.rb

@@ -0,0 +1,32 @@
+module Kamal::Commands::App::Proxy
+  delegate :container_name, to: :"config.proxy_boot", prefix: :proxy
+
+  def deploy(target:)
+    proxy_exec :deploy, role.container_prefix, *role.proxy.deploy_command_args(target: target)
+  end
+
+  def remove
+    proxy_exec :remove, role.container_prefix
+  end
+
+  def live
+    proxy_exec :resume, role.container_prefix
+  end
+
+  def maintenance(**options)
+    proxy_exec :stop, role.container_prefix, *role.proxy.stop_command_args(**options)
+  end
+
+  def remove_proxy_app_directory
+    remove_directory config.proxy_boot.app_directory
+  end
+
+  def create_ssl_directory
+    make_directory(File.join(config.proxy_boot.tls_directory, role.name))
+  end
+
+  private
+    def proxy_exec(*command)
+      docker :exec, proxy_container_name, "kamal-proxy", *command
+    end
+end

lib/kamal/commands/auditor.rb

@@ -1,5 +1,6 @@
 class Kamal::Commands::Auditor < Kamal::Commands::Base
   attr_reader :details
+  delegate :escape_shell_value, to: Kamal::Utils
 
   def initialize(config, **details)
     super(config)
@@ -8,9 +9,9 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
 
   # Runs remotely
   def record(line, **details)
-    append \
-      [ :echo, audit_tags(**details).except(:version, :service_version).to_s, line ],
-      audit_log_file
+    combine \
+      make_run_directory,
+      append([ :echo, escape_shell_value(audit_line(line, **details)) ], audit_log_file)
   end
 
   def reveal
@@ -21,10 +22,18 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
     def audit_log_file
       file = [ config.service, config.destination, "audit.log" ].compact.join("-")
 
-      "#{config.run_directory}/#{file}"
+      File.join(config.run_directory, file)
     end
 
     def audit_tags(**details)
       tags(**self.details, **details)
     end
+
+    def make_run_directory
+      [ :mkdir, "-p", config.run_directory ]
+    end
+
+    def audit_line(line, **details)
+      "#{audit_tags(**details).except(:version, :service_version, :service)} #{line}"
+    end
 end

lib/kamal/commands/base.rb

@@ -3,7 +3,6 @@ module Kamal::Commands
     delegate :sensitive, :argumentize, to: Kamal::Utils
 
     DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'"
-    DOCKER_HEALTH_LOG_FORMAT    = "'{{json .State.Health}}'"
 
     attr_accessor :config
 
@@ -12,14 +11,7 @@ module Kamal::Commands
     end
 
     def run_over_ssh(*command, host:)
-      "ssh".tap do |cmd|
-        if config.ssh.proxy && config.ssh.proxy.is_a?(Net::SSH::Proxy::Jump)
-          cmd << " -J #{config.ssh.proxy.jump_proxies}"
-        elsif config.ssh.proxy && config.ssh.proxy.is_a?(Net::SSH::Proxy::Command)
-          cmd << " -o ProxyCommand='#{config.ssh.proxy.command_line_template}'"
-        end
-        cmd << " -t #{config.ssh.user}@#{host} -p #{config.ssh.port} '#{command.join(" ")}'"
-      end
+      "ssh#{ssh_proxy_args}#{ssh_keys_args} -t #{config.ssh.user}@#{host} -p #{config.ssh.port} '#{command.join(" ").gsub("'", "'\\\\''")}'"
     end
 
     def container_id_for(container_name:, only_running: false)
@@ -38,6 +30,16 @@ module Kamal::Commands
       [ :rm, "-r", path ]
     end
 
+    def remove_file(path)
+      [ :rm, path ]
+    end
+
+    def ensure_docker_installed
+      combine \
+        ensure_local_docker_installed,
+        ensure_local_buildx_installed
+    end
+
     private
       def combine(*commands, by: "&&")
         commands
@@ -66,24 +68,67 @@ module Kamal::Commands
         combine *commands, by: "||"
       end
 
+      def substitute(*commands)
+        "\$\(#{commands.join(" ")}\)"
+      end
+
       def xargs(command)
         [ :xargs, command ].flatten
       end
 
       def shell(command)
-        [ :sh, "-c", "'#{command.flatten.join(" ").gsub("'", "'\\''")}'" ]
+        [ :sh, "-c", "'#{command.flatten.join(" ").gsub("'", "'\\\\''")}'" ]
       end
 
       def docker(*args)
         args.compact.unshift :docker
       end
 
-      def git(*args)
-        args.compact.unshift :git
+      def pack(*args)
+        args.compact.unshift :pack
+      end
+
+      def git(*args, path: nil)
+        [ :git, *([ "-C", path ] if path), *args.compact ]
+      end
+
+      def grep(*args)
+        args.compact.unshift :grep
       end
 
       def tags(**details)
         Kamal::Tags.from_config(config, **details)
       end
+
+      def ssh_proxy_args
+        case config.ssh.proxy
+        when Net::SSH::Proxy::Jump
+          " -J #{config.ssh.proxy.jump_proxies}"
+        when Net::SSH::Proxy::Command
+          " -o ProxyCommand='#{config.ssh.proxy.command_line_template}'"
+        end
+      end
+
+      def ssh_keys_args
+        "#{ ssh_keys.join("") if ssh_keys}" + "#{" -o IdentitiesOnly=yes" if config.ssh&.keys_only}"
+      end
+
+      def ssh_keys
+        config.ssh.keys&.map do |key|
+          " -i #{key}"
+        end
+      end
+
+      def ensure_local_docker_installed
+        docker "--version"
+      end
+
+      def ensure_local_buildx_installed
+        docker :buildx, "version"
+      end
+
+      def docker_interactive_args
+        STDIN.isatty ? "-it" : "-i"
+      end
   end
 end

lib/kamal/commands/builder.rb

@@ -1,64 +1,48 @@
 require "active_support/core_ext/string/filters"
 
 class Kamal::Commands::Builder < Kamal::Commands::Base
-  delegate :create, :remove, :push, :clean, :pull, :info, :validate_image, to: :target
+  delegate :create, :remove, :dev, :push, :clean, :pull, :info, :inspect_builder, :validate_image, :first_mirror, to: :target
+  delegate :local?, :remote?, :pack?, :cloud?, to: "config.builder"
+
+  include Clone
 
   def name
     target.class.to_s.remove("Kamal::Commands::Builder::").underscore.inquiry
   end
 
   def target
-    case
-    when !config.builder.multiarch? && !config.builder.cached?
-      native
-    when !config.builder.multiarch? && config.builder.cached?
-      native_cached
-    when config.builder.local? && config.builder.remote?
-      multiarch_remote
-    when config.builder.remote?
-      native_remote
+    if remote?
+      if local?
+        hybrid
+      else
+        remote
+      end
+    elsif pack?
+      pack
+    elsif cloud?
+      cloud
     else
-      multiarch
+      local
     end
   end
 
-  def native
-    @native ||= Kamal::Commands::Builder::Native.new(config)
+  def remote
+    @remote ||= Kamal::Commands::Builder::Remote.new(config)
   end
 
-  def native_cached
-    @native ||= Kamal::Commands::Builder::Native::Cached.new(config)
+  def local
+    @local ||= Kamal::Commands::Builder::Local.new(config)
   end
 
-  def native_remote
-    @native ||= Kamal::Commands::Builder::Native::Remote.new(config)
+  def hybrid
+    @hybrid ||= Kamal::Commands::Builder::Hybrid.new(config)
   end
 
-  def multiarch
-    @multiarch ||= Kamal::Commands::Builder::Multiarch.new(config)
+  def pack
+    @pack ||= Kamal::Commands::Builder::Pack.new(config)
   end
 
-  def multiarch_remote
-    @multiarch_remote ||= Kamal::Commands::Builder::Multiarch::Remote.new(config)
+  def cloud
+    @cloud ||= Kamal::Commands::Builder::Cloud.new(config)
   end
-
-
-  def ensure_local_dependencies_installed
-    if name.native?
-      ensure_local_docker_installed
-    else
-      combine \
-        ensure_local_docker_installed,
-        ensure_local_buildx_installed
-    end
-  end
-
-  private
-    def ensure_local_docker_installed
-      docker "--version"
-    end
-
-    def ensure_local_buildx_installed
-      docker :buildx, "version"
-    end
 end

lib/kamal/commands/builder/base.rb

@@ -1,34 +1,52 @@
+require "shellwords"
 
 class Kamal::Commands::Builder::Base < Kamal::Commands::Base
   class BuilderError < StandardError; end
 
+  ENDPOINT_DOCKER_HOST_INSPECT = "'{{.Endpoints.docker.Host}}'"
+
   delegate :argumentize, to: Kamal::Utils
-  delegate :args, :secrets, :dockerfile, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, :ssh, :git_archive?, to: :builder_config
+  delegate \
+    :args, :secrets, :dockerfile, :target, :arches, :local_arches, :remote_arches, :remote,
+    :pack?, :pack_builder, :pack_buildpacks,
+    :cache_from, :cache_to, :ssh, :provenance, :sbom, :driver, :docker_driver?,
+    to: :builder_config
 
   def clean
     docker :image, :rm, "--force", config.absolute_image
   end
 
+  def push(export_action = "registry", tag_as_dirty: false)
+    docker :buildx, :build,
+      "--output=type=#{export_action}",
+      *platform_options(arches),
+      *([ "--builder", builder_name ] unless docker_driver?),
+      *build_tag_options(tag_as_dirty: tag_as_dirty),
+      *build_options,
+      build_context,
+      "2>&1"
+  end
+
   def pull
     docker :pull, config.absolute_image
   end
 
-  def push
-    if git_archive?
-      pipe \
-        git(:archive, "--format=tar", :HEAD),
-        build_and_push
-    else
-      build_and_push
-    end
+  def info
+    combine \
+      docker(:context, :ls),
+      docker(:buildx, :ls)
+  end
+
+  def inspect_builder
+    docker :buildx, :inspect, builder_name unless docker_driver?
   end
 
   def build_options
-    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_ssh ]
+    [ *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_target, *build_ssh, *builder_provenance, *builder_sbom ]
   end
 
   def build_context
-    config.builder.context
+    Shellwords.escape(config.builder.context)
   end
 
   def validate_image
@@ -40,10 +58,19 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
       )
   end
 
+  def first_mirror
+    docker(:info, "--format '{{index .RegistryConfig.Mirrors 0}}'")
+  end
 
   private
-    def build_tags
-      [ "-t", config.absolute_image, "-t", config.latest_image ]
+    def build_tag_names(tag_as_dirty: false)
+      tag_names = [ config.absolute_image, config.latest_image ]
+      tag_names.map! { |t| "#{t}-dirty" } if tag_as_dirty
+      tag_names
+    end
+
+    def build_tag_options(tag_as_dirty: false)
+      build_tag_names(tag_as_dirty: tag_as_dirty).flat_map { |name| [ "-t", name ] }
     end
 
     def build_cache
@@ -62,22 +89,38 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
     end
 
     def build_secrets
-      argumentize "--secret", secrets.collect { |secret| [ "id", secret ] }
+      argumentize "--secret", secrets.keys.collect { |secret| [ "id", secret ] }
     end
 
     def build_dockerfile
       if Pathname.new(File.expand_path(dockerfile)).exist?
-        argumentize "--file", dockerfile
+        argumentize "--file", Shellwords.escape(dockerfile)
       else
         raise BuilderError, "Missing #{dockerfile}"
       end
     end
 
+    def build_target
+      argumentize "--target", target if target.present?
+    end
+
     def build_ssh
       argumentize "--ssh", ssh if ssh.present?
     end
 
+    def builder_provenance
+      argumentize "--provenance", provenance unless provenance.nil?
+    end
+
+    def builder_sbom
+      argumentize "--sbom", sbom unless sbom.nil?
+    end
+
     def builder_config
       config.builder
     end
+
+    def platform_options(arches)
+      argumentize "--platform", arches.map { |arch| "linux/#{arch}" }.join(",") if arches.any?
+    end
 end

lib/kamal/commands/builder/clone.rb

@@ -0,0 +1,31 @@
+module Kamal::Commands::Builder::Clone
+  def clone
+    git :clone, escaped_root, "--recurse-submodules", path: config.builder.clone_directory.shellescape
+  end
+
+  def clone_reset_steps
+    [
+      git(:remote, "set-url", :origin, escaped_root, path: escaped_build_directory),
+      git(:fetch, :origin, path: escaped_build_directory),
+      git(:reset, "--hard", Kamal::Git.revision, path: escaped_build_directory),
+      git(:clean, "-fdx", path: escaped_build_directory),
+      git(:submodule, :update, "--init", path: escaped_build_directory)
+    ]
+  end
+
+  def clone_status
+    git :status, "--porcelain", path: escaped_build_directory
+  end
+
+  def clone_revision
+    git :"rev-parse", :HEAD, path: escaped_build_directory
+  end
+
+  def escaped_root
+    Kamal::Git.root.shellescape
+  end
+
+  def escaped_build_directory
+    config.builder.build_directory.shellescape
+  end
+end

lib/kamal/commands/builder/cloud.rb

@@ -0,0 +1,22 @@
+class Kamal::Commands::Builder::Cloud < Kamal::Commands::Builder::Base
+  # Expects `driver` to be of format "cloud docker-org-name/builder-name"
+
+  def create
+    docker :buildx, :create, "--driver", driver
+  end
+
+  def remove
+    docker :buildx, :rm, builder_name
+  end
+
+  private
+    def builder_name
+      driver.gsub(/[ \/]/, "-")
+    end
+
+    def inspect_buildx
+      pipe \
+        docker(:buildx, :inspect, builder_name),
+        grep("-q", "Endpoint:.*cloud://.*")
+    end
+end

lib/kamal/commands/builder/hybrid.rb

@@ -0,0 +1,21 @@
+class Kamal::Commands::Builder::Hybrid < Kamal::Commands::Builder::Remote
+  def create
+    combine \
+      create_local_buildx,
+      create_remote_context,
+      append_remote_buildx
+  end
+
+  private
+    def builder_name
+      "kamal-hybrid-#{driver}-#{remote.gsub(/[^a-z0-9_-]/, "-")}"
+    end
+
+    def create_local_buildx
+      docker :buildx, :create, *platform_options(local_arches), "--name", builder_name, "--driver=#{driver}"
+    end
+
+    def append_remote_buildx
+      docker :buildx, :create, *platform_options(remote_arches), "--append", "--name", builder_name, remote_context_name
+    end
+end

lib/kamal/commands/builder/local.rb

@@ -0,0 +1,27 @@
+class Kamal::Commands::Builder::Local < Kamal::Commands::Builder::Base
+  def create
+    return if docker_driver?
+
+    options =
+      if KAMAL.registry.local?
+        "--driver=#{driver} --driver-opt network=host"
+      else
+        "--driver=#{driver}"
+      end
+
+    docker :buildx, :create, "--name", builder_name, options
+  end
+
+  def remove
+    docker :buildx, :rm, builder_name unless docker_driver?
+  end
+
+  private
+    def builder_name
+      if KAMAL.registry.local?
+        "kamal-local-registry-#{driver}"
+      else
+        "kamal-local-#{driver}"
+      end
+    end
+end

lib/kamal/commands/builder/multiarch.rb

@@ -1,37 +0,0 @@
-class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
-  def create
-    docker :buildx, :create, "--use", "--name", builder_name
-  end
-
-  def remove
-    docker :buildx, :rm, builder_name
-  end
-
-  def info
-    combine \
-      docker(:context, :ls),
-      docker(:buildx, :ls)
-  end
-
-  private
-    def builder_name
-      "kamal-#{config.service}-multiarch"
-    end
-
-    def platform_names
-      if local_arch
-        "linux/#{local_arch}"
-      else
-        "linux/amd64,linux/arm64"
-      end
-    end
-
-    def build_and_push
-      docker :buildx, :build,
-        "--push",
-        "--platform", platform_names,
-        "--builder", builder_name,
-        *build_options,
-        build_context
-    end
-end

lib/kamal/commands/builder/multiarch/remote.rb

@@ -1,51 +0,0 @@
-class Kamal::Commands::Builder::Multiarch::Remote < Kamal::Commands::Builder::Multiarch
-  def create
-    combine \
-      create_contexts,
-      create_local_buildx,
-      append_remote_buildx
-  end
-
-  def remove
-    combine \
-      remove_contexts,
-      super
-  end
-
-  private
-    def builder_name
-      super + "-remote"
-    end
-
-    def builder_name_with_arch(arch)
-      "#{builder_name}-#{arch}"
-    end
-
-    def create_local_buildx
-      docker :buildx, :create, "--name", builder_name, builder_name_with_arch(local_arch), "--platform", "linux/#{local_arch}"
-    end
-
-    def append_remote_buildx
-      docker :buildx, :create, "--append", "--name", builder_name, builder_name_with_arch(remote_arch), "--platform", "linux/#{remote_arch}"
-    end
-
-    def create_contexts
-      combine \
-        create_context(local_arch, local_host),
-        create_context(remote_arch, remote_host)
-    end
-
-    def create_context(arch, host)
-      docker :context, :create, builder_name_with_arch(arch), "--description", "'#{builder_name} #{arch} native host'", "--docker", "'host=#{host}'"
-    end
-
-    def remove_contexts
-      combine \
-        remove_context(local_arch),
-        remove_context(remote_arch)
-    end
-
-    def remove_context(arch)
-      docker :context, :rm, builder_name_with_arch(arch)
-    end
-end

lib/kamal/commands/builder/native.rb

@@ -1,21 +0,0 @@
-class Kamal::Commands::Builder::Native < Kamal::Commands::Builder::Base
-  def create
-    # No-op on native without cache
-  end
-
-  def remove
-    # No-op on native without cache
-  end
-
-  def info
-    # No-op on native
-  end
-
-  private
-    def build_and_push
-      combine \
-        docker(:build, *build_options, build_context),
-        docker(:push, config.absolute_image),
-        docker(:push, config.latest_image)
-    end
-end

lib/kamal/commands/builder/native/cached.rb

@@ -1,17 +0,0 @@
-class Kamal::Commands::Builder::Native::Cached < Kamal::Commands::Builder::Native
-  def create
-    docker :buildx, :create, "--use", "--driver=docker-container"
-  end
-
-  def remove
-    docker :buildx, :rm, builder_name
-  end
-
-  private
-    def build_and_push
-      docker :buildx, :build,
-        "--push",
-        *build_options,
-        build_context
-    end
-end

lib/kamal/commands/builder/native/remote.rb

@@ -1,59 +0,0 @@
-class Kamal::Commands::Builder::Native::Remote < Kamal::Commands::Builder::Native
-  def create
-    chain \
-      create_context,
-      create_buildx
-  end
-
-  def remove
-    chain \
-      remove_context,
-      remove_buildx
-  end
-
-  def info
-    chain \
-      docker(:context, :ls),
-      docker(:buildx, :ls)
-  end
-
-
-  private
-    def builder_name
-      "kamal-#{config.service}-native-remote"
-    end
-
-    def builder_name_with_arch
-      "#{builder_name}-#{remote_arch}"
-    end
-
-    def platform
-      "linux/#{remote_arch}"
-    end
-
-    def create_context
-      docker :context, :create,
-        builder_name_with_arch, "--description", "'#{builder_name} #{remote_arch} native host'", "--docker", "'host=#{remote_host}'"
-    end
-
-    def remove_context
-      docker :context, :rm, builder_name_with_arch
-    end
-
-    def create_buildx
-      docker :buildx, :create, "--name", builder_name, builder_name_with_arch, "--platform", platform
-    end
-
-    def remove_buildx
-      docker :buildx, :rm, builder_name
-    end
-
-    def build_and_push
-      docker :buildx, :build,
-      "--push",
-      "--platform", platform,
-      "--builder", builder_name,
-      *build_options,
-      build_context
-    end
-end

lib/kamal/commands/builder/pack.rb

@@ -0,0 +1,46 @@
+class Kamal::Commands::Builder::Pack < Kamal::Commands::Builder::Base
+  def push(export_action = "registry")
+    combine \
+      build,
+      export(export_action)
+  end
+
+  def remove;end
+
+  def info
+    pack :builder, :inspect, pack_builder
+  end
+  alias_method :inspect_builder, :info
+
+  private
+    def build
+      pack(:build,
+        config.repository,
+        "--platform", platform,
+        "--creation-time", "now",
+        "--builder", pack_builder,
+        buildpacks,
+        "-t", config.absolute_image,
+        "-t", config.latest_image,
+        "--env", "BP_IMAGE_LABELS=service=#{config.service}",
+        *argumentize("--env", args),
+        *argumentize("--env", secrets, sensitive: true),
+        "--path", build_context)
+    end
+
+    def export(export_action)
+      return unless export_action == "registry"
+
+      combine \
+        docker(:push, config.absolute_image),
+        docker(:push, config.latest_image)
+    end
+
+    def platform
+      "linux/#{local_arches.first}"
+    end
+
+    def buildpacks
+      (pack_buildpacks << "paketo-buildpacks/image-labels").map { |buildpack| [ "--buildpack", buildpack ] }
+    end
+end

lib/kamal/commands/builder/remote.rb

@@ -0,0 +1,63 @@
+class Kamal::Commands::Builder::Remote < Kamal::Commands::Builder::Base
+  def create
+    chain \
+      create_remote_context,
+      create_buildx
+  end
+
+  def remove
+    chain \
+      remove_remote_context,
+      remove_buildx
+  end
+
+  def info
+    chain \
+      docker(:context, :ls),
+      docker(:buildx, :ls)
+  end
+
+  def inspect_builder
+    combine \
+      combine(inspect_buildx, inspect_remote_context),
+      [ "(echo no compatible builder && exit 1)" ],
+      by: "||"
+  end
+
+  private
+    def builder_name
+      "kamal-remote-#{remote.gsub(/[^a-z0-9_-]/, "-")}"
+    end
+
+    def remote_context_name
+      "#{builder_name}-context"
+    end
+
+    def inspect_buildx
+      pipe \
+        docker(:buildx, :inspect, builder_name),
+        grep("-q", "Endpoint:.*#{remote_context_name}")
+    end
+
+    def inspect_remote_context
+      pipe \
+        docker(:context, :inspect, remote_context_name, "--format", ENDPOINT_DOCKER_HOST_INSPECT),
+        grep("-xq", remote)
+    end
+
+    def create_remote_context
+      docker :context, :create, remote_context_name, "--description", "'#{builder_name} host'", "--docker", "'host=#{remote}'"
+    end
+
+    def remove_remote_context
+      docker :context, :rm, remote_context_name
+    end
+
+    def create_buildx
+      docker :buildx, :create, "--name", builder_name, remote_context_name
+    end
+
+    def remove_buildx
+      docker :buildx, :rm, builder_name
+    end
+end

lib/kamal/commands/docker.rb

@@ -19,6 +19,10 @@ class Kamal::Commands::Docker < Kamal::Commands::Base
     [ '[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null' ]
   end
 
+  def create_network
+    docker :network, :create, :kamal
+  end
+
   private
     def get_docker
       shell \

lib/kamal/commands/healthcheck.rb

@@ -1,59 +0,0 @@
-class Kamal::Commands::Healthcheck < Kamal::Commands::Base
-  def run
-    primary = config.role(config.primary_role)
-
-    docker :run,
-      "--detach",
-      "--name", container_name_with_version,
-      "--publish", "#{exposed_port}:#{config.healthcheck["port"]}",
-      "--label", "service=#{config.healthcheck_service}",
-      "-e", "KAMAL_CONTAINER_NAME=\"#{config.healthcheck_service}\"",
-      *primary.env_args,
-      *primary.health_check_args(cord: false),
-      *config.volume_args,
-      *primary.option_args,
-      config.absolute_image,
-      primary.cmd
-  end
-
-  def status
-    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_STATUS_FORMAT))
-  end
-
-  def container_health_log
-    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT))
-  end
-
-  def logs
-    pipe container_id, xargs(docker(:logs, "--tail", log_lines, "2>&1"))
-  end
-
-  def stop
-    pipe container_id, xargs(docker(:stop))
-  end
-
-  def remove
-    pipe container_id, xargs(docker(:container, :rm))
-  end
-
-  private
-    def container_name_with_version
-      "#{config.healthcheck_service}-#{config.version}"
-    end
-
-    def container_id
-      container_id_for(container_name: container_name_with_version)
-    end
-
-    def health_url
-      "http://localhost:#{exposed_port}#{config.healthcheck["path"]}"
-    end
-
-    def exposed_port
-      config.healthcheck["exposed_port"]
-    end
-
-    def log_lines
-      config.healthcheck["log_lines"]
-    end
-end

lib/kamal/commands/hook.rb

@@ -1,6 +1,12 @@
 class Kamal::Commands::Hook < Kamal::Commands::Base
-  def run(hook, **details)
-    [ hook_file(hook), env: tags(**details).env ]
+  def run(hook)
+    [ hook_file(hook) ]
+  end
+
+  def env(secrets: false, **details)
+    tags(**details).env.tap do |env|
+      env.merge!(config.secrets.to_h) if secrets
+    end
   end
 
   def hook_exists?(hook)
@@ -9,6 +15,6 @@ class Kamal::Commands::Hook < Kamal::Commands::Base
 
   private
     def hook_file(hook)
-      "#{config.hooks_path}/#{hook}"
+      File.join(config.hooks_path, hook)
     end
 end

lib/kamal/commands/lock.rb

@@ -44,14 +44,10 @@ class Kamal::Commands::Lock < Kamal::Commands::Base
         "/dev/null"
     end
 
-    def locks_dir
-      File.join(config.run_directory, "locks")
-    end
-
     def lock_dir
-      dir_name = [ config.service, config.destination ].compact.join("-")
+      dir_name = [ "lock", config.service, config.destination ].compact.join("-")
 
-      File.join(locks_dir, dir_name)
+      File.join(config.run_directory, dir_name)
     end
 
     def lock_details_file

lib/kamal/commands/proxy.rb

@@ -0,0 +1,127 @@
+class Kamal::Commands::Proxy < Kamal::Commands::Base
+  delegate :argumentize, :optionize, to: Kamal::Utils
+
+  def run
+    pipe boot_config, xargs(docker_run)
+  end
+
+  def start
+    docker :container, :start, container_name
+  end
+
+  def stop(name: container_name)
+    docker :container, :stop, name
+  end
+
+  def start_or_run
+    combine start, run, by: "||"
+  end
+
+  def info
+    docker :ps, "--filter", "name=^#{container_name}$"
+  end
+
+  def version
+    pipe \
+      docker(:inspect, container_name, "--format '{{.Config.Image}}'"),
+      [ :awk, "-F:", "'{print \$NF}'" ]
+  end
+
+  def logs(timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil)
+    pipe \
+      docker(:logs, container_name, ("--since #{since}" if since), ("--tail #{lines}" if lines), ("--timestamps" if timestamps), "2>&1"),
+      ("grep '#{grep}'#{" #{grep_options}" if grep_options}" if grep)
+  end
+
+  def follow_logs(host:, timestamps: true, grep: nil, grep_options: nil)
+    run_over_ssh pipe(
+      docker(:logs, container_name, ("--timestamps" if timestamps), "--tail", "10", "--follow", "2>&1"),
+      (%(grep "#{grep}"#{" #{grep_options}" if grep_options}) if grep)
+    ).join(" "), host: host
+  end
+
+  def remove_container
+    docker :container, :prune, "--force", "--filter", "label=org.opencontainers.image.title=kamal-proxy"
+  end
+
+  def remove_image
+    docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=kamal-proxy"
+  end
+
+  def cleanup_traefik
+    chain \
+      docker(:container, :stop, "traefik"),
+      combine(
+        docker(:container, :prune, "--force", "--filter", "label=org.opencontainers.image.title=Traefik"),
+        docker(:image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=Traefik")
+      )
+  end
+
+  def ensure_proxy_directory
+    make_directory config.proxy_boot.host_directory
+  end
+
+  def remove_proxy_directory
+    remove_directory config.proxy_boot.host_directory
+  end
+
+  def ensure_apps_config_directory
+    make_directory config.proxy_boot.apps_directory
+  end
+
+  def boot_config
+    [ :echo, "#{substitute(read_boot_options)} #{substitute(read_image)}:#{substitute(read_image_version)} #{substitute(read_run_command)}" ]
+  end
+
+  def read_boot_options
+    read_file(config.proxy_boot.options_file, default: config.proxy_boot.default_boot_options.join(" "))
+  end
+
+  def read_image
+    read_file(config.proxy_boot.image_file, default: config.proxy_boot.image_default)
+  end
+
+  def read_image_version
+    read_file(config.proxy_boot.image_version_file, default: Kamal::Configuration::Proxy::Boot::MINIMUM_VERSION)
+  end
+
+  def read_run_command
+    read_file(config.proxy_boot.run_command_file)
+  end
+
+  def reset_boot_options
+    remove_file config.proxy_boot.options_file
+  end
+
+  def reset_image
+    remove_file config.proxy_boot.image_file
+  end
+
+  def reset_image_version
+    remove_file config.proxy_boot.image_version_file
+  end
+
+  def reset_run_command
+    remove_file config.proxy_boot.run_command_file
+  end
+
+  private
+    def container_name
+      config.proxy_boot.container_name
+    end
+
+    def read_file(file, default: nil)
+      combine [ :cat, file, "2>", "/dev/null" ], [ :echo, "\"#{default}\"" ], by: "||"
+    end
+
+    def docker_run
+      docker \
+        :run,
+        "--name", container_name,
+        "--network", "kamal",
+        "--detach",
+        "--restart", "unless-stopped",
+        "--volume", "kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy",
+        *config.proxy_boot.apps_volume.docker_args
+    end
+end

lib/kamal/commands/prune.rb

@@ -9,7 +9,7 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
   def tagged_images
     pipe \
       docker(:image, :ls, *service_filter, "--format", "'{{.ID}} {{.Repository}}:{{.Tag}}'"),
-      "grep -v -w \"#{active_image_list}\"",
+      grep("-v -w \"#{active_image_list}\""),
       "while read image tag; do docker rmi $tag; done"
   end
 
@@ -20,10 +20,6 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
       "while read container_id; do docker rm $container_id; done"
   end
 
-  def healthcheck_containers
-    docker :container, :prune, "--force", *healthcheck_service_filter
-  end
-
   private
     def stopped_containers_filters
       [ "created", "exited", "dead" ].flat_map { |status| [ "--filter", "status=#{status}" ] }
@@ -39,8 +35,4 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
     def service_filter
       [ "--filter", "label=service=#{config.service}" ]
     end
-
-    def healthcheck_service_filter
-      [ "--filter", "label=service=#{config.healthcheck_service}" ]
-    end
 end

lib/kamal/commands/registry.rb

@@ -1,23 +1,38 @@
 class Kamal::Commands::Registry < Kamal::Commands::Base
-  delegate :registry, to: :config
+  def login(registry_config: nil)
+    registry_config ||= config.registry
+
+    return if registry_config.local?
 
-  def login
     docker :login,
-      registry["server"],
-      "-u", sensitive(Kamal::Utils.escape_shell_value(lookup("username"))),
-      "-p", sensitive(Kamal::Utils.escape_shell_value(lookup("password")))
+      registry_config.server,
+      "-u", sensitive(Kamal::Utils.escape_shell_value(registry_config.username)),
+      "-p", sensitive(Kamal::Utils.escape_shell_value(registry_config.password))
   end
 
-  def logout
-    docker :logout, registry["server"]
+  def logout(registry_config: nil)
+    registry_config ||= config.registry
+
+    docker :logout, registry_config.server
   end
 
-  private
-    def lookup(key)
-      if registry[key].is_a?(Array)
-        ENV.fetch(registry[key].first).dup
-      else
-        registry[key]
-      end
-    end
+  def setup(registry_config: nil)
+    registry_config ||= config.registry
+
+    combine \
+      docker(:start, "kamal-docker-registry"),
+      docker(:run, "--detach", "-p", "127.0.0.1:#{registry_config.local_port}:5000", "--name", "kamal-docker-registry", "registry:3"),
+      by: "||"
+  end
+
+  def remove
+    combine \
+      docker(:stop, "kamal-docker-registry"),
+      docker(:rm, "kamal-docker-registry"),
+      by: "&&"
+  end
+
+  def local?
+    config.registry.local?
+  end
 end

lib/kamal/commands/server.rb

@@ -1,5 +1,15 @@
 class Kamal::Commands::Server < Kamal::Commands::Base
   def ensure_run_directory
-    [ :mkdir, "-p", config.run_directory ]
+    make_directory config.run_directory
+  end
+
+  def remove_app_directory
+    remove_directory config.app_directory
+  end
+
+  def app_directory_count
+    pipe \
+      [ :ls, config.apps_directory ],
+      [ :wc, "-l" ]
   end
 end

lib/kamal/commands/traefik.rb

@@ -1,124 +0,0 @@
-class Kamal::Commands::Traefik < Kamal::Commands::Base
-  delegate :argumentize, :optionize, to: Kamal::Utils
-
-  DEFAULT_IMAGE = "traefik:v2.10"
-  CONTAINER_PORT = 80
-  DEFAULT_ARGS = {
-    "log.level" => "DEBUG"
-  }
-  DEFAULT_LABELS = {
-    # These ensure we serve a 502 rather than a 404 if no containers are available
-    "traefik.http.routers.catchall.entryPoints" => "http",
-    "traefik.http.routers.catchall.rule" => "PathPrefix(`/`)",
-    "traefik.http.routers.catchall.service" => "unavailable",
-    "traefik.http.routers.catchall.priority" => 1,
-    "traefik.http.services.unavailable.loadbalancer.server.port" => "0"
-  }
-
-  def run
-    docker :run, "--name traefik",
-      "--detach",
-      "--restart", "unless-stopped",
-      *publish_args,
-      "--volume", "/var/run/docker.sock:/var/run/docker.sock",
-      *env_args,
-      *config.logging_args,
-      *label_args,
-      *docker_options_args,
-      image,
-      "--providers.docker",
-      *cmd_option_args
-  end
-
-  def start
-    docker :container, :start, "traefik"
-  end
-
-  def stop
-    docker :container, :stop, "traefik"
-  end
-
-  def start_or_run
-    any start, run
-  end
-
-  def info
-    docker :ps, "--filter", "name=^traefik$"
-  end
-
-  def logs(since: nil, lines: nil, grep: nil)
-    pipe \
-      docker(:logs, "traefik", (" --since #{since}" if since), (" --tail #{lines}" if lines), "--timestamps", "2>&1"),
-      ("grep '#{grep}'" if grep)
-  end
-
-  def follow_logs(host:, grep: nil)
-    run_over_ssh pipe(
-      docker(:logs, "traefik", "--timestamps", "--tail", "10", "--follow", "2>&1"),
-      (%(grep "#{grep}") if grep)
-    ).join(" "), host: host
-  end
-
-  def remove_container
-    docker :container, :prune, "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
-  end
-
-  def remove_image
-    docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
-  end
-
-  def port
-    "#{host_port}:#{CONTAINER_PORT}"
-  end
-
-  def env
-    Kamal::Configuration::Env.from_config \
-      config: config.traefik.fetch("env", {}),
-      secrets_file: File.join(config.host_env_directory, "traefik", "traefik.env")
-  end
-
-  def make_env_directory
-    make_directory(env.secrets_directory)
-  end
-
-  def remove_env_file
-    [ :rm, "-f", env.secrets_file ]
-  end
-
-  private
-    def publish_args
-      argumentize "--publish", port unless config.traefik["publish"] == false
-    end
-
-    def label_args
-      argumentize "--label", labels
-    end
-
-    def env_args
-      env.args
-    end
-
-    def labels
-      DEFAULT_LABELS.merge(config.traefik["labels"] || {})
-    end
-
-    def image
-      config.traefik.fetch("image") { DEFAULT_IMAGE }
-    end
-
-    def docker_options_args
-      optionize(config.traefik["options"] || {})
-    end
-
-    def cmd_option_args
-      if args = config.traefik["args"]
-        optionize DEFAULT_ARGS.merge(args), with: "="
-      else
-        optionize DEFAULT_ARGS, with: "="
-      end
-    end
-
-    def host_port
-      config.traefik["host_port"] || CONTAINER_PORT
-    end
-end

lib/kamal/configuration.rb

@@ -1,18 +1,23 @@
 require "active_support/ordered_options"
 require "active_support/core_ext/string/inquiry"
 require "active_support/core_ext/module/delegation"
-require "pathname"
+require "active_support/core_ext/hash/keys"
 require "erb"
 require "net/ssh/proxy/jump"
 
 class Kamal::Configuration
-  delegate :service, :image, :servers, :labels, :registry, :stop_wait_time, :hooks_path, :logging, to: :raw_config, allow_nil: true
+  delegate :service, :labels, :hooks_path, to: :raw_config, allow_nil: true
   delegate :argumentize, :optionize, to: Kamal::Utils
 
-  attr_reader :destination, :raw_config
+  attr_reader :destination, :raw_config, :secrets
+  attr_reader :accessories, :aliases, :boot, :builder, :env, :logging, :proxy, :proxy_boot, :servers, :ssh, :sshkit, :registry
+
+  include Validation
 
   class << self
     def create_from(config_file:, destination: nil, version: nil)
+      ENV["KAMAL_DESTINATION"] = destination
+
       raw_config = load_config_files(config_file, *destination_config_file(config_file, destination))
 
       new raw_config, destination: destination, version: version
@@ -27,7 +32,7 @@ class Kamal::Configuration
         if file.exist?
           # Newer Psych doesn't load aliases by default
           load_method = YAML.respond_to?(:unsafe_load) ? :unsafe_load : :load
-          YAML.send(load_method, ERB.new(IO.read(file)).result).symbolize_keys
+          YAML.send(load_method, ERB.new(File.read(file)).result).symbolize_keys
         else
           raise "Configuration file not found in #{file}"
         end
@@ -42,9 +47,36 @@ class Kamal::Configuration
     @raw_config = ActiveSupport::InheritableOptions.new(raw_config)
     @destination = destination
     @declared_version = version
-    valid? if validate
-  end
 
+    validate! raw_config, example: validation_yml.symbolize_keys, context: "", with: Kamal::Configuration::Validator::Configuration
+
+    @secrets = Kamal::Secrets.new(destination: destination)
+
+    # Eager load config to validate it, these are first as they have dependencies later on
+    @servers = Servers.new(config: self)
+    @registry = Registry.new(config: @raw_config, secrets: secrets)
+
+    @accessories = @raw_config.accessories&.keys&.collect { |name| Accessory.new(name, config: self) } || []
+    @aliases = @raw_config.aliases&.keys&.to_h { |name| [ name, Alias.new(name, config: self) ] } || {}
+    @boot = Boot.new(config: self)
+    @builder = Builder.new(config: self)
+    @env = Env.new(config: @raw_config.env || {}, secrets: secrets)
+
+    @logging = Logging.new(logging_config: @raw_config.logging)
+    @proxy = Proxy.new(config: self, proxy_config: @raw_config.proxy, secrets: secrets)
+    @proxy_boot = Proxy::Boot.new(config: self)
+    @ssh = Ssh.new(config: self)
+    @sshkit = Sshkit.new(config: self)
+
+    ensure_destination_if_required
+    ensure_required_keys_present
+    ensure_valid_kamal_version
+    ensure_retain_containers_valid
+    ensure_valid_service_name
+    ensure_no_traefik_reboot_hooks
+    ensure_one_host_for_ssl_roles
+    ensure_unique_hosts_for_ssl_roles
+  end
 
   def version=(version)
     @declared_version = version
@@ -69,25 +101,27 @@ class Kamal::Configuration
     raw_config.minimum_version
   end
 
+  def service_and_destination
+    [ service, destination ].compact.join("-")
+  end
 
   def roles
-    @roles ||= role_names.collect { |role_name| Role.new(role_name, config: self) }
+    servers.roles
   end
 
   def role(name)
     roles.detect { |r| r.name == name.to_s }
   end
 
-  def accessories
-    @accessories ||= raw_config.accessories&.keys&.collect { |name| Kamal::Configuration::Accessory.new(name, config: self) } || []
-  end
-
   def accessory(name)
     accessories.detect { |a| a.name == name.to_s }
   end
 
-
   def all_hosts
+    (roles + accessories).flat_map(&:hosts).uniq
+  end
+
+  def app_hosts
     roles.flat_map(&:hosts).uniq
   end
 
@@ -107,20 +141,31 @@ class Kamal::Configuration
     raw_config.allow_empty_roles
   end
 
-  def traefik_roles
-    roles.select(&:running_traefik?)
+  def proxy_roles
+    roles.select(&:running_proxy?)
   end
 
-  def traefik_role_names
-    traefik_roles.flat_map(&:name)
+  def proxy_role_names
+    proxy_roles.flat_map(&:name)
   end
 
-  def traefik_hosts
-    traefik_roles.flat_map(&:hosts).uniq
+  def proxy_accessories
+    accessories.select(&:running_proxy?)
+  end
+
+  def proxy_hosts
+    (proxy_roles.flat_map(&:hosts) + proxy_accessories.flat_map(&:hosts)).uniq
+  end
+
+  def image
+    name = raw_config&.image.presence
+    name ||= raw_config&.service if registry.local?
+
+    name
   end
 
   def repository
-    [ raw_config.registry["server"], image ].compact.join("/")
+    [ registry.server, image ].compact.join("/")
   end
 
   def absolute_image
@@ -128,7 +173,11 @@ class Kamal::Configuration
   end
 
   def latest_image
-    "#{repository}:latest"
+    "#{repository}:#{latest_tag}"
+  end
+
+  def latest_tag
+    [ "latest", *destination ].join("-")
   end
 
   def service_with_version
@@ -143,7 +192,6 @@ class Kamal::Configuration
     raw_config.retain_containers || 5
   end
 
-
   def volume_args
     if raw_config.volumes.present?
       argumentize "--volume", raw_config.volumes
@@ -153,63 +201,39 @@ class Kamal::Configuration
   end
 
   def logging_args
-    if logging.present?
-      optionize({ "log-driver" => logging["driver"] }.compact) +
-        argumentize("--log-opt", logging["options"])
-    else
-      argumentize("--log-opt", { "max-size" => "10m" })
-    end
-  end
-
-
-  def boot
-    Kamal::Configuration::Boot.new(config: self)
-  end
-
-  def builder
-    Kamal::Configuration::Builder.new(config: self)
-  end
-
-  def traefik
-    raw_config.traefik || {}
-  end
-
-  def ssh
-    Kamal::Configuration::Ssh.new(config: self)
-  end
-
-  def sshkit
-    Kamal::Configuration::Sshkit.new(config: self)
-  end
-
-
-  def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }.merge(raw_config.healthcheck || {})
-  end
-
-  def healthcheck_service
-    [ "healthcheck", service, destination ].compact.join("-")
+    logging.args
   end
 
   def readiness_delay
     raw_config.readiness_delay || 7
   end
 
-  def run_id
-    @run_id ||= SecureRandom.hex(16)
+  def deploy_timeout
+    raw_config.deploy_timeout || 30
   end
 
+  def drain_timeout
+    raw_config.drain_timeout || 30
+  end
 
   def run_directory
-    raw_config.run_directory || ".kamal"
+    ".kamal"
   end
 
-  def run_directory_as_docker_volume
-    if Pathname.new(run_directory).absolute?
-      run_directory
-    else
-      File.join "$(pwd)", run_directory
-    end
+  def apps_directory
+    File.join run_directory, "apps"
+  end
+
+  def app_directory
+    File.join apps_directory, service_and_destination
+  end
+
+  def env_directory
+    File.join app_directory, "env"
+  end
+
+  def assets_directory
+    File.join app_directory, "assets"
   end
 
   def hooks_path
@@ -220,18 +244,20 @@ class Kamal::Configuration
     raw_config.asset_path
   end
 
-
-  def host_env_directory
-    "#{run_directory}/env"
+  def error_pages_path
+    raw_config.error_pages_path
   end
 
-  def env
-    raw_config.env || {}
+  def env_tags
+    @env_tags ||= if (tags = raw_config.env["tags"])
+      tags.collect { |name, config| Env::Tag.new(name, config: config, secrets: secrets) }
+    else
+      []
+    end
   end
 
-
-  def valid?
-    ensure_destination_if_required && ensure_required_keys_present && ensure_valid_kamal_version && ensure_retain_containers_valid && ensure_valid_service_name
+  def env_tag(name)
+    env_tags.detect { |t| t.name == name.to_s }
   end
 
   def to_h
@@ -248,12 +274,10 @@ class Kamal::Configuration
       sshkit: sshkit.to_h,
       builder: builder.to_h,
       accessories: raw_config.accessories,
-      logging: logging_args,
-      healthcheck: healthcheck
+      logging: logging_args
     }.compact
   end
 
-
   private
     # Will raise ArgumentError if any required config keys are missing
     def ensure_destination_if_required
@@ -265,30 +289,28 @@ class Kamal::Configuration
     end
 
     def ensure_required_keys_present
-      %i[ service image registry servers ].each do |key|
-        raise ArgumentError, "Missing required configuration for #{key}" unless raw_config[key].present?
+      %i[ service registry ].each do |key|
+        raise Kamal::ConfigurationError, "Missing required configuration for #{key}" unless raw_config[key].present?
       end
 
-      if raw_config.registry["username"].blank?
-        raise ArgumentError, "You must specify a username for the registry in config/deploy.yml"
-      end
+      raise Kamal::ConfigurationError, "Missing required configuration for image" if image.blank?
 
-      if raw_config.registry["password"].blank?
-        raise ArgumentError, "You must specify a password for the registry in config/deploy.yml (or set the ENV variable if that's used)"
-      end
+      if raw_config.servers.nil?
+        raise Kamal::ConfigurationError, "No servers or accessories specified" unless raw_config.accessories.present?
+      else
+        unless role(primary_role_name).present?
+          raise Kamal::ConfigurationError, "The primary_role #{primary_role_name} isn't defined"
+        end
 
-      unless role_names.include?(primary_role_name)
-        raise ArgumentError, "The primary_role #{primary_role_name} isn't defined"
-      end
+        if primary_role.hosts.empty?
+          raise Kamal::ConfigurationError, "No servers specified for the #{primary_role.name} primary_role"
+        end
 
-      if primary_role.hosts.empty?
-        raise ArgumentError, "No servers specified for the #{primary_role.name} primary_role"
-      end
-
-      unless allow_empty_roles?
-        roles.each do |role|
-          if role.hosts.empty?
-            raise ArgumentError, "No servers specified for the #{role.name} role. You can ignore this with allow_empty_roles: true"
+        unless allow_empty_roles?
+          roles.each do |role|
+            if role.hosts.empty?
+              raise Kamal::ConfigurationError, "No servers specified for the #{role.name} role. You can ignore this with allow_empty_roles: true"
+            end
           end
         end
       end
@@ -297,25 +319,49 @@ class Kamal::Configuration
     end
 
     def ensure_valid_service_name
-      raise ArgumentError, "Service name can only include alphanumeric characters, hyphens, and underscores" unless raw_config[:service] =~ /^[a-z0-9_-]+$/
+      raise Kamal::ConfigurationError, "Service name can only include alphanumeric characters, hyphens, and underscores" unless raw_config[:service] =~ /^[a-z0-9_-]+$/i
 
       true
     end
 
     def ensure_valid_kamal_version
       if minimum_version && Gem::Version.new(minimum_version) > Gem::Version.new(Kamal::VERSION)
-        raise ArgumentError, "Current version is #{Kamal::VERSION}, minimum required is #{minimum_version}"
+        raise Kamal::ConfigurationError, "Current version is #{Kamal::VERSION}, minimum required is #{minimum_version}"
       end
 
       true
     end
 
     def ensure_retain_containers_valid
-      raise ArgumentError, "Must retain at least 1 container" if retain_containers < 1
+      raise Kamal::ConfigurationError, "Must retain at least 1 container" if retain_containers < 1
 
       true
     end
 
+    def ensure_no_traefik_reboot_hooks
+      hooks = %w[ pre-traefik-reboot post-traefik-reboot ].select { |hook_file| File.exist?(File.join(hooks_path, hook_file)) }
+
+      if hooks.any?
+        raise Kamal::ConfigurationError, "Found #{hooks.join(", ")}, these should be renamed to (pre|post)-proxy-reboot"
+      end
+
+      true
+    end
+
+    def ensure_one_host_for_ssl_roles
+      roles.each(&:ensure_one_host_for_ssl)
+
+      true
+    end
+
+    def ensure_unique_hosts_for_ssl_roles
+      hosts = roles.select(&:ssl?).flat_map { |role| role.proxy.hosts }
+      duplicates = hosts.tally.filter_map { |host, count| host if count > 1 }
+
+      raise Kamal::ConfigurationError, "Different roles can't share the same host for SSL: #{duplicates.join(", ")}" if duplicates.any?
+
+      true
+    end
 
     def role_names
       raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort
@@ -324,7 +370,7 @@ class Kamal::Configuration
     def git_version
       @git_version ||=
         if Kamal::Git.used?
-          if Kamal::Git.uncommitted_changes.present? && !builder.git_archive?
+          if Kamal::Git.uncommitted_changes.present? && !builder.git_clone?
             uncommitted_suffix = "_uncommitted_#{SecureRandom.hex(8)}"
           end
           [ Kamal::Git.revision, uncommitted_suffix ].compact.join

lib/kamal/configuration/accessory.rb

@@ -1,65 +1,87 @@
 class Kamal::Configuration::Accessory
+  include Kamal::Configuration::Validation
+
+  DEFAULT_NETWORK = "kamal"
+
   delegate :argumentize, :optionize, to: Kamal::Utils
 
-  attr_accessor :name, :specifics
+  attr_reader :name, :env, :proxy, :registry
 
   def initialize(name, config:)
-    @name, @config, @specifics = name.inquiry, config, config.raw_config["accessories"][name]
+    @name, @config, @accessory_config = name.inquiry, config, config.raw_config["accessories"][name]
+
+    validate! \
+      accessory_config,
+      example: validation_yml["accessories"]["mysql"],
+      context: "accessories/#{name}",
+      with: Kamal::Configuration::Validator::Accessory
+
+    ensure_valid_roles
+
+    @env = initialize_env
+    @proxy = initialize_proxy if running_proxy?
+    @registry = initialize_registry if accessory_config["registry"].present?
   end
 
   def service_name
-    specifics["service"] || "#{config.service}-#{name}"
+    accessory_config["service"] || "#{config.service}-#{name}"
   end
 
   def image
-    specifics["image"]
+    [ registry&.server, accessory_config["image"] ].compact.join("/")
   end
 
   def hosts
-    if (specifics.keys & [ "host", "hosts", "roles" ]).size != 1
-      raise ArgumentError, "Specify one of `host`, `hosts` or `roles` for accessory `#{name}`"
-    end
-
-    hosts_from_host || hosts_from_hosts || hosts_from_roles
+    hosts_from_host || hosts_from_hosts || hosts_from_roles || hosts_from_tags
   end
 
   def port
-    if port = specifics["port"]&.to_s
+    if port = accessory_config["port"]&.to_s
       port.include?(":") ? port : "#{port}:#{port}"
     end
   end
 
+  def network_args
+    argumentize "--network", network
+  end
+
   def publish_args
     argumentize "--publish", port if port
   end
 
   def labels
-    default_labels.merge(specifics["labels"] || {})
+    default_labels.merge(accessory_config["labels"] || {})
   end
 
   def label_args
     argumentize "--label", labels
   end
 
-  def env
-    Kamal::Configuration::Env.from_config \
-      config: specifics.fetch("env", {}),
-      secrets_file: File.join(config.host_env_directory, "accessories", "#{service_name}.env")
+  def env_args
+    [ *env.clear_args, *argumentize("--env-file", secrets_path) ]
   end
 
-  def env_args
-    env.args
+  def env_directory
+    File.join(config.env_directory, "accessories")
+  end
+
+  def secrets_io
+    env.secrets_io
+  end
+
+  def secrets_path
+    File.join(config.env_directory, "accessories", "#{name}.env")
   end
 
   def files
-    specifics["files"]&.to_h do |local_to_remote_mapping|
+    accessory_config["files"]&.to_h do |local_to_remote_mapping|
       local_file, remote_file = local_to_remote_mapping.split(":")
       [ expand_local_file(local_file), expand_remote_file(remote_file) ]
     end || {}
   end
 
   def directories
-    specifics["directories"]&.to_h do |host_to_container_mapping|
+    accessory_config["directories"]&.to_h do |host_to_container_mapping|
       host_path, container_path = host_to_container_mapping.split(":")
       [ expand_host_path(host_path), container_path ]
     end || {}
@@ -74,7 +96,7 @@ class Kamal::Configuration::Accessory
   end
 
   def option_args
-    if args = specifics["options"]
+    if args = accessory_config["options"]
       optionize args
     else
       []
@@ -82,11 +104,37 @@ class Kamal::Configuration::Accessory
   end
 
   def cmd
-    specifics["cmd"]
+    accessory_config["cmd"]
+  end
+
+  def running_proxy?
+    accessory_config["proxy"].present?
   end
 
   private
-    attr_accessor :config
+    attr_reader :config, :accessory_config
+
+    def initialize_env
+      Kamal::Configuration::Env.new \
+        config: accessory_config.fetch("env", {}),
+        secrets: config.secrets,
+        context: "accessories/#{name}/env"
+    end
+
+    def initialize_proxy
+      Kamal::Configuration::Proxy.new \
+        config: config,
+        proxy_config: accessory_config["proxy"],
+        context: "accessories/#{name}/proxy",
+        secrets: config.secrets
+    end
+
+    def initialize_registry
+      Kamal::Configuration::Registry.new \
+        config: accessory_config,
+        secrets: config.secrets,
+        context: "accessories/#{name}/registry"
+    end
 
     def default_labels
       { "service" => service_name }
@@ -108,7 +156,7 @@ class Kamal::Configuration::Accessory
     end
 
     def read_dynamic_file(local_file)
-      StringIO.new(ERB.new(IO.read(local_file)).result)
+      StringIO.new(ERB.new(File.read(local_file)).result)
     end
 
     def expand_remote_file(remote_file)
@@ -116,25 +164,25 @@ class Kamal::Configuration::Accessory
     end
 
     def specific_volumes
-      specifics["volumes"] || []
+      accessory_config["volumes"] || []
     end
 
     def remote_files_as_volumes
-      specifics["files"]&.collect do |local_to_remote_mapping|
+      accessory_config["files"]&.collect do |local_to_remote_mapping|
         _, remote_file = local_to_remote_mapping.split(":")
         "#{service_data_directory + remote_file}:#{remote_file}"
       end || []
     end
 
     def remote_directories_as_volumes
-      specifics["directories"]&.collect do |host_to_container_mapping|
+      accessory_config["directories"]&.collect do |host_to_container_mapping|
         host_path, container_path = host_to_container_mapping.split(":")
         [ expand_host_path(host_path), container_path ].join(":")
       end || []
     end
 
     def expand_host_path(host_path)
-      absolute_path?(host_path) ? host_path : "#{service_data_directory}/#{host_path}"
+      absolute_path?(host_path) ? host_path : File.join(service_data_directory, host_path)
     end
 
     def absolute_path?(path)
@@ -146,30 +194,48 @@ class Kamal::Configuration::Accessory
     end
 
     def hosts_from_host
-      if specifics.key?("host")
-        host = specifics["host"]
-        if host
-          [ host ]
-        else
-          raise ArgumentError, "Missing host for accessory `#{name}`"
-        end
-      end
+      [ accessory_config["host"] ] if accessory_config.key?("host")
     end
 
     def hosts_from_hosts
-      if specifics.key?("hosts")
-        hosts = specifics["hosts"]
-        if hosts.is_a?(Array)
-          hosts
-        else
-          raise ArgumentError, "Hosts should be an Array for accessory `#{name}`"
+      accessory_config["hosts"] if accessory_config.key?("hosts")
+    end
+
+    def hosts_from_roles
+      if accessory_config.key?("role")
+       config.role(accessory_config["role"])&.hosts
+      elsif accessory_config.key?("roles")
+        accessory_config["roles"].flat_map { |role| config.role(role)&.hosts }
+      end
+    end
+
+    def hosts_from_tags
+      if accessory_config.key?("tag")
+        extract_hosts_from_config_with_tag(accessory_config["tag"])
+      elsif accessory_config.key?("tags")
+        accessory_config["tags"].flat_map { |tag| extract_hosts_from_config_with_tag(tag) }
+      end
+    end
+
+    def extract_hosts_from_config_with_tag(tag)
+      if (servers_with_roles = config.raw_config.servers).is_a?(Hash)
+        servers_with_roles.flat_map do |role, servers_in_role|
+          servers_in_role.filter_map do |host|
+            host.keys.first if host.is_a?(Hash) && host.values.first.include?(tag)
+          end
         end
       end
     end
 
-    def hosts_from_roles
-      if specifics.key?("roles")
-        specifics["roles"].flat_map { |role| config.role(role).hosts }
+    def network
+      accessory_config["network"] || DEFAULT_NETWORK
+    end
+
+    def ensure_valid_roles
+      if accessory_config["roles"] && (missing_roles = accessory_config["roles"] - config.roles.map(&:name)).any?
+        raise Kamal::ConfigurationError, "accessories/#{name}: unknown roles #{missing_roles.join(", ")}"
+      elsif accessory_config["role"] && !config.role(accessory_config["role"])
+        raise Kamal::ConfigurationError, "accessories/#{name}: unknown role #{accessory_config["role"]}"
       end
     end
 end

lib/kamal/configuration/alias.rb

@@ -0,0 +1,15 @@
+class Kamal::Configuration::Alias
+  include Kamal::Configuration::Validation
+
+  attr_reader :name, :command
+
+  def initialize(name, config:)
+    @name, @command = name.inquiry, config.raw_config["aliases"][name]
+
+    validate! \
+      command,
+      example: validation_yml["aliases"]["uname"],
+      context: "aliases/#{name}",
+      with: Kamal::Configuration::Validator::Alias
+  end
+end

lib/kamal/configuration/boot.rb

@@ -1,20 +1,25 @@
 class Kamal::Configuration::Boot
+  include Kamal::Configuration::Validation
+
+  attr_reader :boot_config, :host_count
+
   def initialize(config:)
-    @options = config.raw_config.boot || {}
+    @boot_config = config.raw_config.boot || {}
     @host_count = config.all_hosts.count
+    validate! boot_config
   end
 
   def limit
-    limit = @options["limit"]
+    limit = boot_config["limit"]
 
     if limit.to_s.end_with?("%")
-      [ @host_count * limit.to_i / 100, 1 ].max
+      [ host_count * limit.to_i / 100, 1 ].max
     else
       limit
     end
   end
 
   def wait
-    @options["wait"]
+    boot_config["wait"]
   end
 end

lib/kamal/configuration/builder.rb

@@ -1,67 +1,109 @@
 class Kamal::Configuration::Builder
-  def initialize(config:)
-    @options = config.raw_config.builder || {}
-    @image = config.image
-    @server = config.registry["server"]
+  include Kamal::Configuration::Validation
 
-    valid?
+  attr_reader :config, :builder_config
+  delegate :image, :service, to: :config
+  delegate :server, to: :"config.registry"
+
+  def initialize(config:)
+    @config = config
+    @builder_config = config.raw_config.builder || {}
+    @image = config.image
+    @server = config.registry.server
+    @service = config.service
+
+    validate! builder_config, with: Kamal::Configuration::Validator::Builder
   end
 
   def to_h
-    @options
+    builder_config
   end
 
-  def multiarch?
-    @options["multiarch"] != false
+  def remote
+    builder_config["remote"]
   end
 
-  def local?
-    !!@options["local"]
+  def arches
+    Array(builder_config.fetch("arch", default_arch))
+  end
+
+  def local_arches
+    @local_arches ||= if local_disabled?
+      []
+    elsif remote
+      arches & [ Kamal::Utils.docker_arch ]
+    else
+      arches
+    end
+  end
+
+  def remote_arches
+    @remote_arches ||= if remote
+      arches - local_arches
+    else
+      []
+    end
   end
 
   def remote?
-    !!@options["remote"]
+    remote_arches.any?
+  end
+
+  def local?
+    !local_disabled? && (arches.empty? || local_arches.any?)
+  end
+
+  def cloud?
+    driver.start_with? "cloud"
   end
 
   def cached?
-    !!@options["cache"]
+    !!builder_config["cache"]
+  end
+
+  def pack?
+    !!builder_config["pack"]
   end
 
   def args
-    @options["args"] || {}
+    builder_config["args"] || {}
   end
 
   def secrets
-    @options["secrets"] || []
+    (builder_config["secrets"] || []).to_h { |key| [ key, config.secrets[key] ] }
   end
 
   def dockerfile
-    @options["dockerfile"] || "Dockerfile"
+    builder_config["dockerfile"] || "Dockerfile"
+  end
+
+  def target
+    builder_config["target"]
   end
 
   def context
-    @options["context"] || (git_archive? ? "-" : ".")
+    builder_config["context"] || "."
   end
 
-  def local_arch
-    @options["local"]["arch"] if local?
+  def driver
+    builder_config.fetch("driver", "docker-container")
   end
 
-  def local_host
-    @options["local"]["host"] if local?
+  def pack_builder
+    builder_config["pack"]["builder"] if pack?
   end
 
-  def remote_arch
-    @options["remote"]["arch"] if remote?
+  def pack_buildpacks
+    builder_config["pack"]["buildpacks"] if pack?
   end
 
-  def remote_host
-    @options["remote"]["host"] if remote?
+  def local_disabled?
+    builder_config["local"] == false
   end
 
   def cache_from
     if cached?
-      case @options["cache"]["type"]
+      case builder_config["cache"]["type"]
       when "gha"
         cache_from_config_for_gha
       when "registry"
@@ -72,7 +114,7 @@ class Kamal::Configuration::Builder
 
   def cache_to
     if cached?
-      case @options["cache"]["type"]
+      case builder_config["cache"]["type"]
       when "gha"
         cache_to_config_for_gha
       when "registry"
@@ -82,26 +124,57 @@ class Kamal::Configuration::Builder
   end
 
   def ssh
-    @options["ssh"]
+    builder_config["ssh"]
   end
 
-  def git_archive?
-    Kamal::Git.used? && @options["context"].nil?
+  def provenance
+    builder_config["provenance"]
+  end
+
+  def sbom
+    builder_config["sbom"]
+  end
+
+  def git_clone?
+    Kamal::Git.used? && builder_config["context"].nil?
+  end
+
+  def clone_directory
+    @clone_directory ||= File.join Dir.tmpdir, "kamal-clones", [ service, pwd_sha ].compact.join("-")
+  end
+
+  def build_directory
+    @build_directory ||=
+      if git_clone?
+        File.join clone_directory, repo_basename, repo_relative_pwd
+      else
+        "."
+      end
+  end
+
+  def docker_driver?
+    driver == "docker"
   end
 
   private
     def valid?
+      if docker_driver?
+        raise ArgumentError, "Invalid builder configuration: the `docker` driver does not not support remote builders" if remote
+        raise ArgumentError, "Invalid builder configuration: the `docker` driver does not not support caching" if cached?
+        raise ArgumentError, "Invalid builder configuration: the `docker` driver does not not support multiple arches" if arches.many?
+      end
+
       if @options["cache"] && @options["cache"]["type"]
         raise ArgumentError, "Invalid cache type: #{@options["cache"]["type"]}" unless [ "gha", "registry" ].include?(@options["cache"]["type"])
       end
     end
 
     def cache_image
-      @options["cache"]&.fetch("image", nil) || "#{@image}-build-cache"
+      builder_config["cache"]&.fetch("image", nil) || "#{image}-build-cache"
     end
 
     def cache_image_ref
-      [ @server, cache_image ].compact.join("/")
+      [ server, cache_image ].compact.join("/")
     end
 
     def cache_from_config_for_gha
@@ -113,10 +186,26 @@ class Kamal::Configuration::Builder
     end
 
     def cache_to_config_for_gha
-      [ "type=gha", @options["cache"]&.fetch("options", nil) ].compact.join(",")
+      [ "type=gha", builder_config["cache"]&.fetch("options", nil) ].compact.join(",")
     end
 
     def cache_to_config_for_registry
-      [ "type=registry", @options["cache"]&.fetch("options", nil), "ref=#{cache_image_ref}" ].compact.join(",")
+      [ "type=registry", "ref=#{cache_image_ref}", builder_config["cache"]&.fetch("options", nil) ].compact.join(",")
+    end
+
+    def repo_basename
+      File.basename(Kamal::Git.root)
+    end
+
+    def repo_relative_pwd
+      Dir.pwd.delete_prefix(Kamal::Git.root)
+    end
+
+    def pwd_sha
+      Digest::SHA256.hexdigest(Dir.pwd)[0..12]
+    end
+
+    def default_arch
+      docker_driver? ? [] : [ "amd64", "arm64" ]
     end
 end

lib/kamal/configuration/docs/accessory.yml

@@ -0,0 +1,128 @@
+# Accessories
+#
+# Accessories can be booted on a single host, a list of hosts, or on specific roles.
+# The hosts do not need to be defined in the Kamal servers configuration.
+#
+# Accessories are managed separately from the main service — they are not updated
+# when you deploy, and they do not have zero-downtime deployments.
+#
+# Run `kamal accessory boot <accessory>` to boot an accessory.
+# See `kamal accessory --help` for more information.
+
+# Configuring accessories
+#
+# First, define the accessory in the `accessories`:
+accessories:
+  mysql:
+
+    # Service name
+    #
+    # This is used in the service label and defaults to `<service>-<accessory>`,
+    # where `<service>` is the main service name from the root configuration:
+    service: mysql
+
+    # Image
+    #
+    # The Docker image to use.
+    # Prefix it with its server when using root level registry different from Docker Hub.
+    # Define registry directly or via anchors when it differs from root level registry.
+    image: mysql:8.0
+
+    # Registry
+    #
+    # By default accessories use Docker Hub registry.
+    # You can specify different registry per accessory with this option.
+    # Don't prefix image with this registry server.
+    # Use anchors if you need to set the same specific registry for several accessories.
+    #
+    # ```yml
+    # registry:
+    #   <<: *specific-registry
+    # ```
+    #
+    # See kamal docs registry for more information:
+    registry:
+      ...
+
+    # Accessory hosts
+    #
+    # Specify one of `host`, `hosts`, `role`, `roles`, `tag` or `tags`:
+    host: mysql-db1
+    hosts:
+      - mysql-db1
+      - mysql-db2
+    role: mysql
+    roles:
+      - mysql
+    tag: writer
+    tags:
+      - writer
+      - reader
+
+    # Custom command
+    #
+    # You can set a custom command to run in the container if you do not want to use the default:
+    cmd: "bin/mysqld"
+
+    # Port mappings
+    #
+    # See [https://docs.docker.com/network/](https://docs.docker.com/network/), and
+    # especially note the warning about the security implications of exposing ports publicly.
+    port: "127.0.0.1:3306:3306"
+
+    # Labels
+    labels:
+      app: myapp
+
+    # Options
+    #
+    # These are passed to the Docker run command in the form `--<name> <value>`:
+    options:
+      restart: always
+      cpus: 2
+
+    # Environment variables
+    #
+    # See kamal docs env for more information:
+    env:
+      ...
+
+    # Copying files
+    #
+    # You can specify files to mount into the container.
+    # The format is `local:remote`, where `local` is the path to the file on the local machine
+    # and `remote` is the path to the file in the container.
+    #
+    # They will be uploaded from the local repo to the host and then mounted.
+    #
+    # ERB files will be evaluated before being copied.
+    files:
+      - config/my.cnf.erb:/etc/mysql/my.cnf
+      - config/myoptions.cnf:/etc/mysql/myoptions.cnf
+
+    # Directories
+    #
+    # You can specify directories to mount into the container. They will be created on the host
+    # before being mounted:
+    directories:
+      - mysql-logs:/var/log/mysql
+
+    # Volumes
+    #
+    # Any other volumes to mount, in addition to the files and directories.
+    # They are not created or copied before mounting:
+    volumes:
+      - /path/to/mysql-logs:/var/log/mysql
+
+    # Network
+    #
+    # The network the accessory will be attached to.
+    #
+    # Defaults to kamal:
+    network: custom
+
+    # Proxy
+    #
+    # You can run your accessory behind the Kamal proxy. See kamal docs proxy for more information
+    proxy:
+      ...

lib/kamal/configuration/docs/alias.yml

@@ -0,0 +1,26 @@
+# Aliases
+#
+# Aliases are shortcuts for Kamal commands.
+#
+# For example, for a Rails app, you might open a console with:
+#
+# ```shell
+# kamal app exec -i --reuse "bin/rails console"
+# ```
+#
+# By defining an alias, like this:
+aliases:
+  console: app exec -i --reuse "bin/rails console"
+# You can now open the console with:
+#
+# ```shell
+# kamal console
+# ```
+
+# Configuring aliases
+#
+# Aliases are defined in the root config under the alias key.
+#
+# Each alias is named and can only contain lowercase letters, numbers, dashes, and underscores:
+aliases:
+  uname: app exec -p -q -r web "uname -a"

lib/kamal/configuration/docs/boot.yml

@@ -0,0 +1,19 @@
+# Booting
+#
+# When deploying to large numbers of hosts, you might prefer not to restart your services on every host at the same time.
+#
+# Kamal’s default is to boot new containers on all hosts in parallel. However, you can control this with the boot configuration.
+
+# Fixed group sizes
+#
+# Here, we boot 2 hosts at a time with a 10-second gap between each group:
+boot:
+  limit: 2
+  wait: 10
+
+# Percentage of hosts
+#
+# Here, we boot 25% of the hosts at a time with a 2-second gap between each group:
+boot:
+  limit: 25%
+  wait: 2

lib/kamal/configuration/docs/builder.yml

@@ -0,0 +1,132 @@
+# Builder
+#
+# The builder configuration controls how the application is built with `docker build`.
+#
+# See https://kamal-deploy.org/docs/configuration/builder-examples/ for more information.
+
+# Builder options
+#
+# Options go under the builder key in the root configuration.
+builder:
+
+  # Arch
+  #
+  # The architectures to build for — you can set an array or just a single value.
+  #
+  # Allowed values are `amd64` and `arm64`:
+  arch:
+    - amd64
+
+  # Remote
+  #
+  # The connection string for a remote builder. If supplied, Kamal will use this
+  # for builds that do not match the local architecture of the deployment host.
+  remote: ssh://docker@docker-builder
+
+  # Local
+  #
+  # If set to false, Kamal will always use the remote builder even when building
+  # the local architecture.
+  #
+  # Defaults to true:
+  local: true
+
+  # Buildpack configuration
+  #
+  # The build configuration for using pack to build a Cloud Native Buildpack image.
+  #
+  # For additional buildpack customization options you can create a project descriptor
+  # file(project.toml) that the Pack CLI will automatically use.
+  # See https://buildpacks.io/docs/for-app-developers/how-to/build-inputs/use-project-toml/ for more information.
+  pack:
+    builder: heroku/builder:24
+    buildpacks:
+      - heroku/ruby
+      - heroku/procfile
+
+  # Builder cache
+  #
+  # The type must be either 'gha' or 'registry'.
+  #
+  # The image is only used for registry cache and is not compatible with the Docker driver:
+  cache:
+    type: registry
+    options: mode=max
+    image: kamal-app-build-cache
+
+  # Build context
+  #
+  # If this is not set, then a local Git clone of the repo is used.
+  # This ensures a clean build with no uncommitted changes.
+  #
+  # To use the local checkout instead, you can set the context to `.`, or a path to another directory.
+  context: .
+
+  # Dockerfile
+  #
+  # The Dockerfile to use for building, defaults to `Dockerfile`:
+  dockerfile: Dockerfile.production
+
+  # Build target
+  #
+  # If not set, then the default target is used:
+  target: production
+
+  # Build arguments
+  #
+  # Any additional build arguments, passed to `docker build` with `--build-arg <key>=<value>`:
+  args:
+    ENVIRONMENT: production
+
+  # Referencing build arguments
+  #
+  # ```shell
+  # ARG RUBY_VERSION
+  # FROM ruby:$RUBY_VERSION-slim as base
+  # ```
+
+  # Build secrets
+  #
+  # Values are read from `.kamal/secrets`:
+  secrets:
+    - SECRET1
+    - SECRET2
+
+  # Referencing build secrets
+  #
+  # ```shell
+  # # Copy Gemfiles
+  # COPY Gemfile Gemfile.lock ./
+  #
+  # # Install dependencies, including private repositories via access token
+  # # Then remove bundle cache with exposed GITHUB_TOKEN
+  # RUN --mount=type=secret,id=GITHUB_TOKEN \
+  #   BUNDLE_GITHUB__COM=x-access-token:$(cat /run/secrets/GITHUB_TOKEN) \
+  #   bundle install && \
+  #   rm -rf /usr/local/bundle/cache
+  # ```
+
+  # SSH
+  #
+  # SSH agent socket or keys to expose to the build:
+  ssh: default=$SSH_AUTH_SOCK
+
+  # Driver
+  #
+  # The build driver to use, defaults to `docker-container`:
+  driver: docker
+  #
+  # If you want to use Docker Build Cloud (https://www.docker.com/products/build-cloud/), you can set the driver to:
+  driver: cloud org-name/builder-name
+
+  # Provenance
+  #
+  # It is used to configure provenance attestations for the build result.
+  # The value can also be a boolean to enable or disable provenance attestations.
+  provenance: mode=max
+
+  # SBOM (Software Bill of Materials)
+  #
+  # It is used to configure SBOM generation for the build result.
+  # The value can also be a boolean to enable or disable SBOM generation.
+  sbom: true

lib/kamal/configuration/docs/configuration.yml

@@ -0,0 +1,184 @@
+# Kamal Configuration
+#
+# Configuration is read from the `config/deploy.yml`.
+
+# Destinations
+#
+# When running commands, you can specify a destination with the `-d` flag,
+# e.g., `kamal deploy -d staging`.
+#
+# In this case, the configuration will also be read from `config/deploy.staging.yml`
+# and merged with the base configuration.
+
+# Extensions
+#
+# Kamal will not accept unrecognized keys in the configuration file.
+#
+# However, you might want to declare a configuration block using YAML anchors
+# and aliases to avoid repetition.
+#
+# You can prefix a configuration section with `x-` to indicate that it is an
+# extension. Kamal will ignore the extension and not raise an error.
+
+# The service name
+#
+# This is a required value. It is used as the container name prefix.
+service: myapp
+
+# The Docker image name
+#
+# The image will be pushed to the configured registry.
+image: my-image
+
+# Labels
+#
+# Additional labels to add to the container:
+labels:
+  my-label: my-value
+
+# Volumes
+#
+# Additional volumes to mount into the container:
+volumes:
+  - /path/on/host:/path/in/container:ro
+
+# Registry
+#
+# The Docker registry configuration, see kamal docs registry:
+registry:
+  ...
+
+# Servers
+#
+# The servers to deploy to, optionally with custom roles, see kamal docs servers:
+servers:
+  ...
+
+# Environment variables
+#
+# See kamal docs env:
+env:
+  ...
+
+# Asset path
+#
+# Used for asset bridging across deployments, default to `nil`.
+#
+# If there are changes to CSS or JS files, we may get requests
+# for the old versions on the new container, and vice versa.
+#
+# To avoid 404s, we can specify an asset path.
+# Kamal will replace that path in the container with a mapped
+# volume containing both sets of files.
+# This requires that file names change when the contents change
+# (e.g., by including a hash of the contents in the name).
+#
+# To configure this, set the path to the assets:
+asset_path: /path/to/assets
+
+# Hooks path
+#
+# Path to hooks, defaults to `.kamal/hooks`.
+# See https://kamal-deploy.org/docs/hooks for more information:
+hooks_path: /user_home/kamal/hooks
+
+# Error pages
+#
+# A directory relative to the app root to find error pages for the proxy to serve.
+# Any files in the format 4xx.html or 5xx.html will be copied to the hosts.
+error_pages_path: public
+
+# Require destinations
+#
+# Whether deployments require a destination to be specified, defaults to `false`:
+require_destination: true
+
+# Primary role
+#
+# This defaults to `web`, but if you have no web role, you can change this:
+primary_role: workers
+
+# Allowing empty roles
+#
+# Whether roles with no servers are allowed. Defaults to `false`:
+allow_empty_roles: false
+
+# Retain containers
+#
+# How many old containers and images we retain, defaults to 5:
+retain_containers: 3
+
+# Minimum version
+#
+# The minimum version of Kamal required to deploy this configuration, defaults to `nil`:
+minimum_version: 1.3.0
+
+# Readiness delay
+#
+# Seconds to wait for a container to boot after it is running, default 7.
+#
+# This only applies to containers that do not run a proxy or specify a healthcheck:
+readiness_delay: 4
+
+# Deploy timeout
+#
+# How long to wait for a container to become ready, default 30:
+deploy_timeout: 10
+
+# Drain timeout
+#
+# How long to wait for a container to drain, default 30:
+drain_timeout: 10
+
+# Run directory
+#
+# Directory to store kamal runtime files in on the host, default `.kamal`:
+run_directory: /etc/kamal
+
+# SSH options
+#
+# See kamal docs ssh:
+ssh:
+  ...
+
+# Builder options
+#
+# See kamal docs builder:
+builder:
+  ...
+
+# Accessories
+#
+# Additional services to run in Docker, see kamal docs accessory:
+accessories:
+  ...
+
+# Proxy
+#
+# Configuration for kamal-proxy, see kamal docs proxy:
+proxy:
+  ...
+
+# SSHKit
+#
+# See kamal docs sshkit:
+sshkit:
+  ...
+
+# Boot options
+#
+# See kamal docs boot:
+boot:
+  ...
+
+# Logging
+#
+# Docker logging configuration, see kamal docs logging:
+logging:
+  ...
+
+# Aliases
+#
+# Alias configuration, see kamal docs alias:
+aliases:
+  ...

lib/kamal/configuration/docs/env.yml

@@ -0,0 +1,116 @@
+# Environment variables
+#
+# Environment variables can be set directly in the Kamal configuration or
+# read from `.kamal/secrets`.
+
+# Reading environment variables from the configuration
+#
+# Environment variables can be set directly in the configuration file.
+#
+# These are passed to the `docker run` command when deploying.
+env:
+  DATABASE_HOST: mysql-db1
+  DATABASE_PORT: 3306
+
+# Secrets
+#
+# Kamal uses dotenv to automatically load environment variables set in the `.kamal/secrets` file.
+#
+# If you are using destinations, secrets will instead be read from `.kamal/secrets.<DESTINATION>` if
+# it exists.
+#
+# Common secrets across all destinations can be set in `.kamal/secrets-common`.
+#
+# This file can be used to set variables like `KAMAL_REGISTRY_PASSWORD` or database passwords.
+# You can use variable or command substitution in the secrets file.
+#
+# ```shell
+# KAMAL_REGISTRY_PASSWORD=$KAMAL_REGISTRY_PASSWORD
+# RAILS_MASTER_KEY=$(cat config/master.key)
+# ```
+#
+# You can also use [secret helpers](../../commands/secrets) for some common password managers.
+#
+# ```shell
+# SECRETS=$(kamal secrets fetch ...)
+#
+# REGISTRY_PASSWORD=$(kamal secrets extract REGISTRY_PASSWORD $SECRETS)
+# DB_PASSWORD=$(kamal secrets extract DB_PASSWORD $SECRETS)
+# ```
+#
+# If you store secrets directly in `.kamal/secrets`, ensure that it is not checked into version control.
+#
+# To pass the secrets, you should list them under the `secret` key. When you do this, the
+# other variables need to be moved under the `clear` key.
+#
+# Unlike clear values, secrets are not passed directly to the container
+# but are stored in an env file on the host:
+env:
+  clear:
+    DB_USER: app
+  secret:
+    - DB_PASSWORD
+
+# Aliased secrets
+#
+# You can also alias secrets to other secrets using a `:` separator.
+#
+# This is useful when the ENV name is different from the secret name. For example, if you have two
+# places where you need to define the ENV variable `DB_PASSWORD`, but the value is different depending
+# on the context.
+#
+# ```shell
+# SECRETS=$(kamal secrets fetch ...)
+#
+# MAIN_DB_PASSWORD=$(kamal secrets extract MAIN_DB_PASSWORD $SECRETS)
+# SECONDARY_DB_PASSWORD=$(kamal secrets extract SECONDARY_DB_PASSWORD $SECRETS)
+# ```
+env:
+  secret:
+    - DB_PASSWORD:MAIN_DB_PASSWORD
+  tags:
+    secondary_db:
+      secret:
+        - DB_PASSWORD:SECONDARY_DB_PASSWORD
+accessories:
+  main_db_accessory:
+    env:
+      secret:
+        - DB_PASSWORD:MAIN_DB_PASSWORD
+  secondary_db_accessory:
+    env:
+      secret:
+        - DB_PASSWORD:SECONDARY_DB_PASSWORD
+
+# Tags
+#
+# Tags are used to add extra env variables to specific hosts.
+# See kamal docs servers for how to tag hosts.
+#
+# Tags are only allowed in the top-level env configuration (i.e., not under a role-specific env).
+#
+# The env variables can be specified with secret and clear values as explained above.
+env:
+  tags:
+    <tag1>:
+      MYSQL_USER: monitoring
+    <tag2>:
+      clear:
+        MYSQL_USER: readonly
+      secret:
+        - MYSQL_PASSWORD
+
+# Example configuration
+env:
+  clear:
+    MYSQL_USER: app
+  secret:
+    - MYSQL_PASSWORD
+  tags:
+    monitoring:
+      MYSQL_USER: monitoring
+    replica:
+      clear:
+        MYSQL_USER: readonly
+      secret:
+        - READONLY_PASSWORD

lib/kamal/configuration/docs/logging.yml

@@ -0,0 +1,21 @@
+# Custom logging configuration
+#
+# Set these to control the Docker logging driver and options.
+
+# Logging settings
+#
+# These go under the logging key in the configuration file.
+#
+# This can be specified at the root level or for a specific role.
+logging:
+
+  # Driver
+  #
+  # The logging driver to use, passed to Docker via `--log-driver`:
+  driver: json-file
+
+  # Options
+  #
+  # Any logging options to pass to the driver, passed to Docker via `--log-opt`:
+  options:
+    max-size: 100m

lib/kamal/configuration/docs/proxy.yml

@@ -0,0 +1,164 @@
+# Proxy
+#
+# Kamal uses [kamal-proxy](https://github.com/basecamp/kamal-proxy) to provide
+# gapless deployments. It runs on ports 80 and 443 and forwards requests to the
+# application container.
+#
+# The proxy is configured in the root configuration under `proxy`. These are
+# options that are set when deploying the application, not when booting the proxy.
+#
+# They are application-specific, so they are not shared when multiple applications
+# run on the same proxy.
+#
+proxy:
+
+  # Hosts
+  #
+  # The hosts that will be used to serve the app. The proxy will only route requests
+  # to this host to your app.
+  #
+  # If no hosts are set, then all requests will be forwarded, except for matching
+  # requests for other apps deployed on that server that do have a host set.
+  #
+  # Specify one of `host` or `hosts`.
+  host: foo.example.com
+  hosts:
+    - foo.example.com
+    - bar.example.com
+
+  # App port
+  #
+  # The port the application container is exposed on.
+  #
+  # Defaults to 80:
+  app_port: 3000
+
+  # SSL
+  #
+  # kamal-proxy can provide automatic HTTPS for your application via Let's Encrypt.
+  #
+  # This requires that we are deploying to one server and the host option is set.
+  # The host value must point to the server we are deploying to, and port 443 must be
+  # open for the Let's Encrypt challenge to succeed.
+  #
+  # If you set `ssl` to `true`, `kamal-proxy` will stop forwarding headers to your app,
+  # unless you explicitly set `forward_headers: true`
+  #
+  # Defaults to `false`:
+  ssl: true
+
+  # Custom SSL certificate
+  #
+  # In some cases, using Let's Encrypt for automatic certificate management is not an
+  # option, for example if you are running from more than one host.
+  #
+  # Or you may already have SSL certificates issued by a different Certificate Authority (CA).
+  #
+  # Kamal supports loading custom SSL certificates directly from secrets. You should
+  # pass a hash mapping the `certificate_pem` and `private_key_pem` to the secret names.
+  ssl:
+    certificate_pem: CERTIFICATE_PEM
+    private_key_pem: PRIVATE_KEY_PEM
+  # ### Notes
+  # - If the certificate or key is missing or invalid, deployments will fail.
+  # - Always handle SSL certificates and private keys securely. Avoid hard-coding them in source control.
+
+  # SSL redirect
+  #
+  # By default, kamal-proxy will redirect all HTTP requests to HTTPS when SSL is enabled.
+  # If you prefer that HTTP traffic is passed through to your application (along with
+  # HTTPS traffic), you can disable this redirect by setting `ssl_redirect: false`:
+  ssl_redirect: false
+
+  # Forward headers
+  #
+  # Whether to forward the `X-Forwarded-For` and `X-Forwarded-Proto` headers.
+  #
+  # If you are behind a trusted proxy, you can set this to `true` to forward the headers.
+  #
+  # By default, kamal-proxy will not forward the headers if the `ssl` option is set to `true`, and
+  # will forward them if it is set to `false`.
+  forward_headers: true
+
+  # Response timeout
+  #
+  # How long to wait for requests to complete before timing out, defaults to 30 seconds:
+  response_timeout: 10
+
+  # Path-based routing
+  #
+  # For applications that split their traffic to different services based on the request path,
+  # you can use path-based routing to mount services under different path prefixes.
+  path_prefix: '/api'
+  # By default, the path prefix will be stripped from the request before it is forwarded upstream.
+  # So in the example above, a request to /api/users/123 will be forwarded to web-1 as /users/123.
+  # To instead forward the request with the original path (including the prefix),
+  # specify --strip-path-prefix=false
+  strip_path_prefix: false
+
+  # Healthcheck
+  #
+  # When deploying, the proxy will by default hit `/up` once every second until we hit
+  # the deploy timeout, with a 5-second timeout for each request.
+  #
+  # Once the app is up, the proxy will stop hitting the healthcheck endpoint.
+  healthcheck:
+    interval: 3
+    path: /health
+    timeout: 3
+
+  # Buffering
+  #
+  # Whether to buffer request and response bodies in the proxy.
+  #
+  # By default, buffering is enabled with a max request body size of 1GB and no limit
+  # for response size.
+  #
+  # You can also set the memory limit for buffering, which defaults to 1MB; anything
+  # larger than that is written to disk.
+  buffering:
+    requests: true
+    responses: true
+    max_request_body: 40_000_000
+    max_response_body: 0
+    memory: 2_000_000
+
+  # Logging
+  #
+  # Configure request logging for the proxy.
+  # You can specify request and response headers to log.
+  # By default, `Cache-Control`, `Last-Modified`, and `User-Agent` request headers are logged:
+  logging:
+    request_headers:
+      - Cache-Control
+      - X-Forwarded-Proto
+    response_headers:
+      - X-Request-ID
+      - X-Request-Start
+
+# Enabling/disabling the proxy on roles
+#
+# The proxy is enabled by default on the primary role but can be disabled by
+# setting `proxy: false` in the primary role's configuration.
+#
+# ```yaml
+# servers:
+#   web:
+#     hosts:
+#      - ...
+#     proxy: false
+# ```
+#
+# It is disabled by default on all other roles but can be enabled by setting
+# `proxy: true` or providing a proxy configuration for that role.
+#
+# ```yaml
+# servers:
+#   web:
+#     hosts:
+#      - ...
+#   web2:
+#     hosts:
+#      - ...
+#     proxy: true
+# ```

lib/kamal/configuration/docs/registry.yml

@@ -0,0 +1,56 @@
+# Registry
+#
+# The default registry is Docker Hub, but you can change it using `registry/server`.
+#
+# By default, Docker Hub creates public repositories. To avoid making your images public,
+# set up a private repository before deploying, or change the default repository privacy
+# settings to private in your [Docker Hub settings](https://hub.docker.com/repository-settings/default-privacy).
+#
+# A reference to a secret (in this case, `DOCKER_REGISTRY_TOKEN`) will look up the secret
+# in the local environment:
+registry:
+  server: registry.digitalocean.com
+  username:
+    - DOCKER_REGISTRY_TOKEN
+  password:
+    - DOCKER_REGISTRY_TOKEN
+
+# Using AWS ECR as the container registry
+#
+# You will need to have the AWS CLI installed locally for this to work.
+# AWS ECR’s access token is only valid for 12 hours. In order to avoid having to manually regenerate the token every time, you can use ERB in the `deploy.yml` file to shell out to the AWS CLI command and obtain the token:
+registry:
+  server: <your aws account id>.dkr.ecr.<your aws region id>.amazonaws.com
+  username: AWS
+  password: <%= %x(aws ecr get-login-password) %>
+
+# Using GCP Artifact Registry as the container registry
+#
+# To sign into Artifact Registry, you need to
+# [create a service account](https://cloud.google.com/iam/docs/service-accounts-create#creating)
+# and [set up roles and permissions](https://cloud.google.com/artifact-registry/docs/access-control#permissions).
+# Normally, assigning the `roles/artifactregistry.writer` role should be sufficient.
+#
+# Once the service account is ready, you need to generate and download a JSON key and base64 encode it:
+#
+# ```shell
+# base64 -i /path/to/key.json | tr -d "\\n"
+# ```
+#
+# You'll then need to set the `KAMAL_REGISTRY_PASSWORD` secret to that value.
+#
+# Use the environment variable as the password along with `_json_key_base64` as the username.
+# Here’s the final configuration:
+registry:
+  server: <your registry region>-docker.pkg.dev
+  username: _json_key_base64
+  password:
+    - KAMAL_REGISTRY_PASSWORD
+
+# Validating the configuration
+#
+# You can validate the configuration by running:
+#
+# ```shell
+# kamal registry login
+# ```

lib/kamal/configuration/docs/role.yml

@@ -0,0 +1,53 @@
+# Roles
+#
+# Roles are used to configure different types of servers in the deployment.
+# The most common use for this is to run web servers and job servers.
+#
+# Kamal expects there to be a `web` role, unless you set a different `primary_role`
+# in the root configuration.
+
+# Role configuration
+#
+# Roles are specified under the servers key:
+servers:
+
+  # Simple role configuration
+  #
+  # This can be a list of hosts if you don't need custom configuration for the role.
+  #
+  # You can set tags on the hosts for custom env variables (see kamal docs env):
+  web:
+    - 172.1.0.1
+    - 172.1.0.2: experiment1
+    - 172.1.0.2: [ experiment1, experiment2 ]
+
+  # Custom role configuration
+  #
+  # When there are other options to set, the list of hosts goes under the `hosts` key.
+  #
+  # By default, only the primary role uses a proxy.
+  #
+  # For other roles, you can set it to `proxy: true` to enable it and inherit the root proxy
+  # configuration or provide a map of options to override the root configuration.
+  #
+  # For the primary role, you can set `proxy: false` to disable the proxy.
+  #
+  # You can also set a custom `cmd` to run in the container and overwrite other settings
+  # from the root configuration.
+  workers:
+    hosts:
+      - 172.1.0.3
+      - 172.1.0.4: experiment1
+    cmd: "bin/jobs"
+    options:
+      memory: 2g
+      cpus: 4
+    logging:
+      ...
+    proxy:
+      ...
+    labels:
+      my-label: workers
+    env:
+      ...
+    asset_path: /public

lib/kamal/configuration/docs/servers.yml

@@ -0,0 +1,27 @@
+# Servers
+#
+# Servers are split into different roles, with each role having its own configuration.
+#
+# For simpler deployments, though, where all servers are identical, you can just specify a list of servers.
+# They will be implicitly assigned to the `web` role.
+servers:
+  - 172.0.0.1
+  - 172.0.0.2
+  - 172.0.0.3
+
+# Tagging servers
+#
+# Servers can be tagged, with the tags used to add custom env variables (see kamal docs env).
+servers:
+  - 172.0.0.1
+  - 172.0.0.2: experiments
+  - 172.0.0.3: [ experiments, three ]
+
+# Roles
+#
+# For more complex deployments (e.g., if you are running job hosts), you can specify roles and configure each separately (see kamal docs role):
+servers:
+  web:
+    ...
+  workers:
+    ...