All count as proxy hosts

This will allow us to share files with the proxy via the host.

.github/workflows/ci.yml

@@ -1,28 +1,42 @@
 name: CI
-on: 
+on:
   push:
     branches:
       - main
   pull_request:
 jobs:
+  rubocop:
+    name: RuboCop
+    runs-on: ubuntu-latest
+    env:
+      BUNDLE_ONLY: rubocop
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Setup Ruby and install gems
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.3.0
+          bundler-cache: true
+      - name: Run Rubocop
+        run: bundle exec rubocop --parallel
   tests:
     strategy:
       matrix:
         ruby-version:
-          - "2.7"
           - "3.1"
           - "3.2"
+          - "3.3"
         gemfile:
           - Gemfile
           - gemfiles/rails_edge.gemfile
-        continue-on-error: [false]
     name: ${{ format('Tests (Ruby {0})', matrix.ruby-version) }}
     runs-on: ubuntu-latest
-    continue-on-error: ${{ matrix.continue-on-error }}
+    continue-on-error: true
     env:
       BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install Ruby
         uses: ruby/setup-ruby@v1

.rubocop.yml

@@ -0,0 +1,2 @@
+inherit_gem:
+  rubocop-rails-omakase: rubocop.yml

Dockerfile

@@ -1,7 +1,7 @@
 # Use the official Ruby 3.2.0 Alpine image as the base image
 FROM ruby:3.2.0-alpine
 
-# Install  docker/buildx-bin
+# Install docker/buildx-bin
 COPY --from=docker/buildx-bin /buildx /usr/libexec/docker/cli-plugins/docker-buildx
 
 # Set the working directory to /kamal
@@ -14,7 +14,7 @@ COPY Gemfile Gemfile.lock kamal.gemspec ./
 COPY lib/kamal/version.rb /kamal/lib/kamal/version.rb
 
 # Install system dependencies
-RUN apk add --no-cache --update build-base git docker openrc openssh-client-default \
+RUN apk add --no-cache build-base git docker openrc openssh-client-default \
     && rc-update add docker boot \
     && gem install bundler --version=2.4.3 \
     && bundle install

Gemfile

@@ -1,4 +1,8 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 
 gemspec
+
+group :rubocop do
+  gem "rubocop-rails-omakase", require: false
+end

Gemfile.lock

@@ -1,96 +1,173 @@
 PATH
   remote: .
   specs:
-    kamal (1.1.0)
+    kamal (2.0.0.alpha)
       activesupport (>= 7.0)
+      base64 (~> 0.2)
       bcrypt_pbkdf (~> 1.0)
       concurrent-ruby (~> 1.2)
-      dotenv (~> 2.8)
+      dotenv (~> 3.1)
       ed25519 (~> 1.2)
       net-ssh (~> 7.0)
-      sshkit (~> 1.21)
+      sshkit (>= 1.23.0, < 2.0)
-      thor (~> 1.2)
+      thor (~> 1.3)
       zeitwerk (~> 2.5)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.0.4.3)
+    actionpack (7.1.3.4)
-      actionview (= 7.0.4.3)
+      actionview (= 7.1.3.4)
-      activesupport (= 7.0.4.3)
+      activesupport (= 7.1.3.4)
-      rack (~> 2.0, >= 2.2.0)
+      nokogiri (>= 1.8.5)
+      racc
+      rack (>= 2.2.4)
+      rack-session (>= 1.0.1)
       rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.0)
+      rails-dom-testing (~> 2.2)
-      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+      rails-html-sanitizer (~> 1.6)
-    actionview (7.0.4.3)
+    actionview (7.1.3.4)
-      activesupport (= 7.0.4.3)
+      activesupport (= 7.1.3.4)
       builder (~> 3.1)
-      erubi (~> 1.4)
+      erubi (~> 1.11)
-      rails-dom-testing (~> 2.0)
+      rails-dom-testing (~> 2.2)
-      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+      rails-html-sanitizer (~> 1.6)
-    activesupport (7.0.4.3)
+    activesupport (7.1.3.4)
+      base64
+      bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
+      mutex_m
       tzinfo (~> 2.0)
-    bcrypt_pbkdf (1.1.0)
+    ast (2.4.2)
-    builder (3.2.4)
+    base64 (0.2.0)
-    concurrent-ruby (1.2.2)
+    bcrypt_pbkdf (1.1.1)
+    bcrypt_pbkdf (1.1.1-arm64-darwin)
+    bcrypt_pbkdf (1.1.1-x86_64-darwin)
+    bigdecimal (3.1.8)
+    builder (3.3.0)
+    concurrent-ruby (1.3.3)
+    connection_pool (2.4.1)
     crass (1.0.6)
-    debug (1.7.2)
+    debug (1.9.2)
-      irb (>= 1.5.0)
+      irb (~> 1.10)
-      reline (>= 0.3.1)
+      reline (>= 0.3.8)
-    dotenv (2.8.1)
+    dotenv (3.1.2)
+    drb (2.2.1)
     ed25519 (1.3.0)
-    erubi (1.12.0)
+    erubi (1.13.0)
-    i18n (1.12.0)
+    i18n (1.14.5)
       concurrent-ruby (~> 1.0)
-    io-console (0.6.0)
+    io-console (0.7.2)
-    irb (1.6.3)
+    irb (1.14.0)
-      reline (>= 0.3.0)
+      rdoc (>= 4.0.0)
-    loofah (2.20.0)
+      reline (>= 0.4.2)
+    json (2.7.2)
+    language_server-protocol (3.17.0.3)
+    loofah (2.22.0)
       crass (~> 1.0.2)
-      nokogiri (>= 1.5.9)
+      nokogiri (>= 1.12.0)
-    method_source (1.0.0)
+    minitest (5.24.1)
-    minitest (5.18.0)
+    mocha (2.4.5)
-    mocha (2.0.2)
       ruby2_keywords (>= 0.0.5)
+    mutex_m (0.2.0)
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
-    net-ssh (7.1.0)
+    net-sftp (4.0.0)
-    nokogiri (1.14.2-arm64-darwin)
+      net-ssh (>= 5.0.0, < 8.0.0)
+    net-ssh (7.2.3)
+    nokogiri (1.16.7-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.2-x86_64-darwin)
+    nokogiri (1.16.7-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.2-x86_64-linux)
+    nokogiri (1.16.7-x86_64-linux)
       racc (~> 1.4)
-    racc (1.6.2)
+    parallel (1.25.1)
-    rack (2.2.6.4)
+    parser (3.3.4.0)
+      ast (~> 2.4.1)
+      racc
+    psych (5.1.2)
+      stringio
+    racc (1.8.1)
+    rack (3.1.7)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
     rack-test (2.1.0)
       rack (>= 1.3)
-    rails-dom-testing (2.0.3)
+    rackup (2.1.0)
-      activesupport (>= 4.2.0)
+      rack (>= 3)
+      webrick (~> 1.8)
+    rails-dom-testing (2.2.0)
+      activesupport (>= 5.0.0)
+      minitest
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.5.0)
+    rails-html-sanitizer (1.6.0)
-      loofah (~> 2.19, >= 2.19.1)
+      loofah (~> 2.21)
-    railties (7.0.4.3)
+      nokogiri (~> 1.14)
-      actionpack (= 7.0.4.3)
+    railties (7.1.3.4)
-      activesupport (= 7.0.4.3)
+      actionpack (= 7.1.3.4)
-      method_source
+      activesupport (= 7.1.3.4)
+      irb
+      rackup (>= 1.0.0)
       rake (>= 12.2)
-      thor (~> 1.0)
+      thor (~> 1.0, >= 1.2.2)
-      zeitwerk (~> 2.5)
+      zeitwerk (~> 2.6)
-    rake (13.0.6)
+    rainbow (3.1.1)
-    reline (0.3.3)
+    rake (13.2.1)
+    rdoc (6.7.0)
+      psych (>= 4.0.0)
+    regexp_parser (2.9.2)
+    reline (0.5.9)
       io-console (~> 0.5)
+    rexml (3.3.4)
+      strscan
+    rubocop (1.65.1)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.4, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.32.0)
+      parser (>= 3.3.1.0)
+    rubocop-minitest (0.35.1)
+      rubocop (>= 1.61, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-performance (1.21.1)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-rails (2.25.1)
+      activesupport (>= 4.2.0)
+      rack (>= 1.1)
+      rubocop (>= 1.33.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-rails-omakase (1.0.0)
+      rubocop
+      rubocop-minitest
+      rubocop-performance
+      rubocop-rails
+    ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
-    sshkit (1.21.4)
+    sshkit (1.23.0)
+      base64
       net-scp (>= 1.1.2)
+      net-sftp (>= 2.1.2)
       net-ssh (>= 2.8.0)
-    thor (1.2.1)
+    stringio (3.1.1)
+    strscan (3.1.0)
+    thor (1.3.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.7)
+    unicode-display_width (2.5.0)
+    webrick (1.8.1)
+    zeitwerk (2.6.17)
 
 PLATFORMS
   arm64-darwin
@@ -102,6 +179,7 @@ DEPENDENCIES
   kamal!
   mocha
   railties
+  rubocop-rails-omakase
 
 BUNDLED WITH
    2.4.3

bin/docs

@@ -0,0 +1,137 @@
+#!/usr/bin/env ruby
+require "stringio"
+
+def usage
+  puts "Usage: #{$0} <kamal_site_repo>"
+  exit 1
+end
+
+usage if ARGV.size != 1
+
+kamal_site_repo = ARGV[0]
+
+if !File.directory?(kamal_site_repo)
+  puts "Error: #{kamal_site_repo} is not a directory"
+  exit 1
+end
+
+DOCS = {
+  "accessory" => "Accessories",
+  "alias" => "Aliases",
+  "boot" => "Booting",
+  "builder" => "Builders",
+  "configuration" => "Configuration overview",
+  "env" => "Environment variables",
+  "healthcheck" => "Healthchecks",
+  "logging" => "Logging",
+  "proxy" => "Proxy (Experimental)",
+  "registry" => "Docker Registry",
+  "role" => "Roles",
+  "servers" => "Servers",
+  "ssh" => "SSH",
+  "sshkit" => "SSHKit",
+  "traefik" => "Traefik"
+}
+
+class DocWriter
+  attr_reader :from_file, :to_file, :key, :heading, :body, :output, :in_yaml
+
+  def initialize(from_file, to_dir)
+    @from_file = from_file
+    @key = File.basename(from_file, ".yml")
+    @to_file = File.join(to_dir, "#{linkify(DOCS[key])}.md")
+    @body = File.readlines(from_file)
+    @heading = body.shift.chomp("\n")
+    @output = nil
+  end
+
+  def write
+    puts "Writing #{to_file}"
+    generate_markdown
+    File.write(to_file, output.string)
+  end
+
+  private
+    def generate_markdown
+      @output = StringIO.new
+
+      generate_header
+
+      place = :in_section
+
+      loop do
+        line = body.shift&.chomp("\n")
+        break if line.nil?
+
+        case place
+        when :new_section, :in_section
+          if line.empty?
+            output.puts
+            place = :new_section
+          elsif line =~ /^ *#/
+            generate_line(line, heading: place == :new_section)
+            place = :in_section
+          else
+            output.puts "```yaml"
+            output.puts line
+            place = :in_yaml
+          end
+        when :in_yaml, :in_empty_line_yaml
+          if line =~ /^ *#/
+            output.puts "```"
+            generate_line(line, heading: place == :in_empty_line_yaml)
+            place = :in_section
+          elsif line.empty?
+            place = :in_empty_line_yaml
+          else
+            output.puts line
+          end
+        end
+      end
+
+      output.puts "```" if place == :in_yaml
+    end
+
+    def generate_header
+      output.puts "---"
+      output.puts "title: #{heading[2..-1]}"
+      output.puts "---"
+      output.puts
+      output.puts heading
+      output.puts
+    end
+
+    def generate_line(line, heading: false)
+      line = line.gsub(/^ *#\s?/, "")
+
+      if line =~ /(.*)kamal docs ([a-z]*)(.*)/
+        line = "#{$1}[#{DOCS[$2]}](../#{linkify(DOCS[$2])})#{$3}"
+      end
+
+      if line =~ /(.*)https:\/\/kamal-deploy.org([a-z\/-]*)(.*)/
+        line = "#{$1}[#{titlify($2.split("/").last)}](#{$2})#{$3}"
+      end
+
+      if heading
+        output.puts "## [#{line}](##{linkify(line)})"
+      else
+        output.puts line
+      end
+    end
+
+    def linkify(text)
+      text.downcase.gsub(" ", "-")
+    end
+
+    def titlify(text)
+      text.capitalize.gsub("-", " ")
+    end
+end
+
+from_dir = File.join(File.dirname(__FILE__), "../lib/kamal/configuration/docs")
+to_dir = File.join(kamal_site_repo, "docs/configuration")
+Dir.glob("#{from_dir}/*") do |from_file|
+  key = File.basename(from_file, ".yml")
+
+  DocWriter.new(from_file, to_dir).write
+end

kamal.gemspec

@@ -12,14 +12,15 @@ Gem::Specification.new do |spec|
   spec.executables = %w[ kamal ]
 
   spec.add_dependency "activesupport", ">= 7.0"
-  spec.add_dependency "sshkit", "~> 1.21"
+  spec.add_dependency "sshkit", ">= 1.23.0", "< 2.0"
   spec.add_dependency "net-ssh", "~> 7.0"
-  spec.add_dependency "thor", "~> 1.2"
+  spec.add_dependency "thor", "~> 1.3"
-  spec.add_dependency "dotenv", "~> 2.8"
+  spec.add_dependency "dotenv", "~> 3.1"
   spec.add_dependency "zeitwerk", "~> 2.5"
   spec.add_dependency "ed25519", "~> 1.2"
   spec.add_dependency "bcrypt_pbkdf", "~> 1.0"
   spec.add_dependency "concurrent-ruby", "~> 1.2"
+  spec.add_dependency "base64", "~> 0.2"
 
   spec.add_development_dependency "debug"
   spec.add_development_dependency "mocha"

lib/kamal.rb

@@ -1,10 +1,14 @@
 module Kamal
+  class ConfigurationError < StandardError; end
 end
 
 require "active_support"
 require "zeitwerk"
+require "yaml"
+require "tmpdir"
+require "pathname"
 
 loader = Zeitwerk::Loader.for_gem
-loader.ignore("#{__dir__}/kamal/sshkit_with_ext.rb")
+loader.ignore(File.join(__dir__, "kamal", "sshkit_with_ext.rb"))
 loader.setup
-loader.eager_load # We need all commands loaded.
+loader.eager_load_namespace(Kamal::Cli) # We need all commands loaded.

lib/kamal/cli.rb

@@ -1,6 +1,7 @@
 module Kamal::Cli
-  class LockError < StandardError; end
+  class BootError < StandardError; end
   class HookError < StandardError; end
+  class LockError < StandardError; end
 end
 
 # SSHKit uses instance eval, so we need a global const for ergonomics

lib/kamal/cli/accessory.rb

@@ -1,17 +1,20 @@
 class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "boot [NAME]", "Boot new accessory service on host (use NAME=all to boot all accessories)"
-  def boot(name, login: true)
+  def boot(name, prepare: true)
-    mutating do
+    with_lock do
       if name == "all"
         KAMAL.accessory_names.each { |accessory_name| boot(accessory_name) }
       else
-        with_accessory(name) do |accessory|
+        prepare(name) if prepare
+
+        with_accessory(name) do |accessory, hosts|
           directories(name)
           upload(name)
 
-          on(accessory.hosts) do
+          on(hosts) do
-            execute *KAMAL.registry.login if login
             execute *KAMAL.auditor.record("Booted #{name} accessory"), verbosity: :debug
+            execute *accessory.ensure_env_directory
+            upload! accessory.secrets_io, accessory.secrets_path, mode: "0600"
             execute *accessory.run
           end
         end
@@ -21,9 +24,9 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "upload [NAME]", "Upload accessory files to host", hide: true
   def upload(name)
-    mutating do
+    with_lock do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           accessory.files.each do |(local, remote)|
             accessory.ensure_local_file_present(local)
 
@@ -38,9 +41,9 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "directories [NAME]", "Create accessory directories on host", hide: true
   def directories(name)
-    mutating do
+    with_lock do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           accessory.directories.keys.each do |host_path|
             execute *accessory.make_directory(host_path)
           end
@@ -49,26 +52,25 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     end
   end
 
-  desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container)"
+  desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container; use NAME=all to boot all accessories)"
   def reboot(name)
-    mutating do
+    with_lock do
-      with_accessory(name) do |accessory|
+      if name == "all"
-        on(accessory.hosts) do
+        KAMAL.accessory_names.each { |accessory_name| reboot(accessory_name) }
-          execute *KAMAL.registry.login
+      else
-        end
+        prepare(name)
-
         stop(name)
         remove_container(name)
-        boot(name, login: false)
+        boot(name, prepare: false)
       end
     end
   end
 
   desc "start [NAME]", "Start existing accessory container on host"
   def start(name)
-    mutating do
+    with_lock do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Started #{name} accessory"), verbosity: :debug
           execute *accessory.start
         end
@@ -78,9 +80,9 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "stop [NAME]", "Stop existing accessory container on host"
   def stop(name)
-    mutating do
+    with_lock do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Stopped #{name} accessory"), verbosity: :debug
           execute *accessory.stop, raise_on_non_zero_exit: false
         end
@@ -90,11 +92,9 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "restart [NAME]", "Restart existing accessory container on host"
   def restart(name)
-    mutating do
+    with_lock do
-      with_accessory(name) do
+      stop(name)
-        stop(name)
+      start(name)
-        start(name)
-      end
     end
   end
 
@@ -103,8 +103,9 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
     if name == "all"
       KAMAL.accessory_names.each { |accessory_name| details(accessory_name) }
     else
-      with_accessory(name) do |accessory|
+      type = "Accessory #{name}"
-        on(accessory.hosts) { puts capture_with_info(*accessory.info) }
+      with_accessory(name) do |accessory, hosts|
+        on(hosts) { puts_by_host host, capture_with_info(*accessory.info), type: type }
       end
     end
   end
@@ -113,7 +114,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
   option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
   def exec(name, cmd)
-    with_accessory(name) do |accessory|
+    with_accessory(name) do |accessory, hosts|
       case
       when options[:interactive] && options[:reuse]
         say "Launching interactive command with via SSH from existing container...", :magenta
@@ -125,14 +126,14 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
       when options[:reuse]
         say "Launching command from existing container...", :magenta
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
           capture_with_info(*accessory.execute_in_existing_container(cmd))
         end
 
       else
         say "Launching command from new container...", :magenta
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug
           capture_with_info(*accessory.execute_in_new_container(cmd))
         end
@@ -144,23 +145,25 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
   option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server"
   option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
+  option :grep_options, aliases: "-o", desc: "Additional options supplied to grep"
   option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
   def logs(name)
-    with_accessory(name) do |accessory|
+    with_accessory(name) do |accessory, hosts|
       grep = options[:grep]
+      grep_options = options[:grep_options]
 
       if options[:follow]
         run_locally do
-          info "Following logs on #{accessory.hosts}..."
+          info "Following logs on #{hosts}..."
-          info accessory.follow_logs(grep: grep)
+          info accessory.follow_logs(grep: grep, grep_options: grep_options)
-          exec accessory.follow_logs(grep: grep)
+          exec accessory.follow_logs(grep: grep, grep_options: grep_options)
         end
       else
         since = options[:since]
         lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
-        on(accessory.hosts) do
+        on(hosts) do
-          puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep))
+          puts capture_with_info(*accessory.logs(since: since, lines: lines, grep: grep, grep_options: grep_options))
         end
       end
     end
@@ -169,17 +172,12 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
   desc "remove [NAME]", "Remove accessory container, image and data directory from host (use NAME=all to remove all accessories)"
   option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def remove(name)
-    mutating do
+    confirming "This will remove all containers, images and data directories for #{name}. Are you sure?" do
-      if name == "all"
+      with_lock do
-        KAMAL.accessory_names.each { |accessory_name| remove(accessory_name) }
+        if name == "all"
-      else
+          KAMAL.accessory_names.each { |accessory_name| remove_accessory(accessory_name) }
-        if options[:confirmed] || ask("This will remove all containers, images and data directories for #{name}. Are you sure?", limited_to: %w( y N ), default: "N") == "y"
+        else
-          with_accessory(name) do
+          remove_accessory(name)
-            stop(name)
-            remove_container(name)
-            remove_image(name)
-            remove_service_directory(name)
-          end
         end
       end
     end
@@ -187,9 +185,9 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "remove_container [NAME]", "Remove accessory container from host", hide: true
   def remove_container(name)
-    mutating do
+    with_lock do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Remove #{name} accessory container"), verbosity: :debug
           execute *accessory.remove_container
         end
@@ -199,9 +197,9 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "remove_image [NAME]", "Remove accessory image from host", hide: true
   def remove_image(name)
-    mutating do
+    with_lock do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *KAMAL.auditor.record("Removed #{name} accessory image"), verbosity: :debug
           execute *accessory.remove_image
         end
@@ -211,9 +209,9 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   desc "remove_service_directory [NAME]", "Remove accessory directory used for uploaded files and data directories from host", hide: true
   def remove_service_directory(name)
-    mutating do
+    with_lock do
-      with_accessory(name) do |accessory|
+      with_accessory(name) do |accessory, hosts|
-        on(accessory.hosts) do
+        on(hosts) do
           execute *accessory.remove_service_directory
         end
       end
@@ -222,8 +220,9 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
 
   private
     def with_accessory(name)
-      if accessory = KAMAL.accessory(name)
+      if KAMAL.config.accessory(name)
-        yield accessory
+        accessory = KAMAL.accessory(name)
+        yield accessory, accessory_hosts(accessory)
       else
         error_on_missing_accessory(name)
       end
@@ -236,4 +235,30 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
         "No accessory by the name of '#{name}'" +
         (options ? " (options: #{options.to_sentence})" : "")
     end
+
+    def accessory_hosts(accessory)
+      if KAMAL.specific_hosts&.any?
+        KAMAL.specific_hosts & accessory.hosts
+      else
+        accessory.hosts
+      end
+    end
+
+    def remove_accessory(name)
+      stop(name)
+      remove_container(name)
+      remove_image(name)
+      remove_service_directory(name)
+    end
+
+    def prepare(name)
+      with_accessory(name) do |accessory, hosts|
+        on(hosts) do
+          execute *KAMAL.registry.login
+          execute *KAMAL.docker.create_network
+        rescue SSHKit::Command::Failed => e
+          raise unless e.message.include?("already exists")
+        end
+      end
+    end
 end

lib/kamal/cli/alias/command.rb

@@ -0,0 +1,9 @@
+class Kamal::Cli::Alias::Command < Thor::DynamicCommand
+  def run(instance, args = [])
+    if (_alias = KAMAL.config.aliases[name])
+      Kamal::Cli::Main.start(Shellwords.split(_alias.command) + ARGV[1..-1])
+    else
+      super
+    end
+  end
+end

lib/kamal/cli/app.rb

@@ -1,80 +1,54 @@
 class Kamal::Cli::App < Kamal::Cli::Base
   desc "boot", "Boot app on servers (or reboot app if already running)"
   def boot
-    mutating do
+    with_lock do
-      hold_lock_on_error do
+      say "Get most recent version available as an image...", :magenta unless options[:version]
-        say "Get most recent version available as an image...", :magenta unless options[:version]
+      using_version(version_or_latest) do |version|
-        using_version(version_or_latest) do |version|
+        say "Start container with version #{version} using a #{KAMAL.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
-          say "Start container with version #{version} using a #{KAMAL.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
 
-          on(KAMAL.hosts) do
+        # Assets are prepared in a separate step to ensure they are on all hosts before booting
-            execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
+        on(KAMAL.hosts) do
-            execute *KAMAL.app.tag_current_image_as_latest
+          KAMAL.roles_on(host).each do |role|
-
+            Kamal::Cli::App::PrepareAssets.new(host, role, self).run
-            KAMAL.roles_on(host).each do |role|
-              app = KAMAL.app(role: role)
-              role_config = KAMAL.config.role(role)
-
-              if role_config.assets?
-                execute *app.extract_assets
-                old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-                execute *app.sync_asset_volumes(old_version: old_version)
-              end
-            end
           end
+        end
 
-          on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
+        # Primary hosts and roles are returned first, so they can open the barrier
-            KAMAL.roles_on(host).each do |role|
+        barrier = Kamal::Cli::Healthcheck::Barrier.new
-              app = KAMAL.app(role: role)
-              auditor = KAMAL.auditor(role: role)
-              role_config = KAMAL.config.role(role)
 
-              if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
+        on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
-                tmp_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
+          KAMAL.roles_on(host).each do |role|
-                info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
+            Kamal::Cli::App::Boot.new(host, role, self, version, barrier).run
-                execute *auditor.record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
-                execute *app.rename_container(version: version, new_version: tmp_version)
-              end
-
-              old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
-
-              execute *app.tie_cord(role_config.cord_host_file) if role_config.uses_cord?
-
-              execute *auditor.record("Booted app version #{version}"), verbosity: :debug
-
-              execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
-
-              Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
-
-              if old_version.present?
-                if role_config.uses_cord?
-                  cord = capture_with_info(*app.cord(version: old_version), raise_on_non_zero_exit: false).strip
-                  if cord.present?
-                    execute *app.cut_cord(cord)
-                    Kamal::Cli::Healthcheck::Poller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
-                  end
-                end
-
-                execute *app.stop(version: old_version), raise_on_non_zero_exit: false
-
-                execute *app.clean_up_assets if role_config.assets?
-              end
-            end
           end
         end
+
+        # Tag once the app booted on all hosts
+        on(KAMAL.hosts) do |host|
+          execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
+          execute *KAMAL.app.tag_latest_image
+        end
       end
     end
   end
 
   desc "start", "Start existing app container on servers"
   def start
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
+          app = KAMAL.app(role: role, host: host)
           execute *KAMAL.auditor.record("Started app version #{KAMAL.config.version}"), verbosity: :debug
-          execute *KAMAL.app(role: role).start, raise_on_non_zero_exit: false
+          execute *app.start, raise_on_non_zero_exit: false
+
+          if role.running_traefik? && KAMAL.proxy_host?(host)
+            version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+            endpoint = capture_with_info(*app.container_id_for_version(version)).strip
+            raise Kamal::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
+
+            execute *KAMAL.proxy.deploy(role.container_prefix, target: endpoint)
+          end
         end
       end
     end
@@ -82,13 +56,23 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "stop", "Stop app container on servers"
   def stop
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
+          app = KAMAL.app(role: role, host: host)
           execute *KAMAL.auditor.record("Stopped app", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).stop, raise_on_non_zero_exit: false
+
+          if role.running_traefik? && KAMAL.proxy_host?(host)
+            version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+            endpoint = capture_with_info(*app.container_id_for_version(version)).strip
+            if endpoint.present?
+              execute *KAMAL.proxy.remove(role.container_prefix, target: endpoint), raise_on_non_zero_exit: false
+            end
+          end
+
+          execute *app.stop, raise_on_non_zero_exit: false
         end
       end
     end
@@ -101,21 +85,24 @@ class Kamal::Cli::App < Kamal::Cli::Base
       roles = KAMAL.roles_on(host)
 
       roles.each do |role|
-        puts_by_host host, capture_with_info(*KAMAL.app(role: role).info)
+        puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).info)
       end
     end
   end
 
-  desc "exec [CMD]", "Execute a custom command on servers (use --help to show options)"
+  desc "exec [CMD...]", "Execute a custom command on servers within the app container (use --help to show options)"
   option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)"
   option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one"
-  def exec(cmd)
+  option :env, aliases: "-e", type: :hash, desc: "Set environment variables for the command"
+  def exec(*cmd)
+    cmd = Kamal::Utils.join_commands(cmd)
+    env = options[:env]
     case
     when options[:interactive] && options[:reuse]
       say "Get current version of running container...", :magenta unless options[:version]
       using_version(options[:version] || current_running_version) do |version|
         say "Launching interactive command with version #{version} via SSH from existing container on #{KAMAL.primary_host}...", :magenta
-        run_locally { exec KAMAL.app(role: KAMAL.primary_role).execute_in_existing_container_over_ssh(cmd, host: KAMAL.primary_host) }
+        run_locally { exec KAMAL.app(role: KAMAL.primary_role, host: KAMAL.primary_host).execute_in_existing_container_over_ssh(cmd, env: env) }
       end
 
     when options[:interactive]
@@ -123,7 +110,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
       using_version(version_or_latest) do |version|
         say "Launching interactive command with version #{version} via SSH from new container on #{KAMAL.primary_host}...", :magenta
         run_locally do
-          exec KAMAL.app(role: KAMAL.primary_role).execute_in_new_container_over_ssh(cmd, host: KAMAL.primary_host)
+          exec KAMAL.app(role: KAMAL.primary_role, host: KAMAL.primary_host).execute_in_new_container_over_ssh(cmd, env: env)
         end
       end
 
@@ -137,7 +124,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
           roles.each do |role|
             execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}", role: role), verbosity: :debug
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_existing_container(cmd))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).execute_in_existing_container(cmd, env: env))
           end
         end
       end
@@ -151,7 +138,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
           roles.each do |role|
             execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_new_container(cmd))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).execute_in_new_container(cmd, env: env))
           end
         end
       end
@@ -166,19 +153,21 @@ class Kamal::Cli::App < Kamal::Cli::Base
   desc "stale_containers", "Detect app stale containers"
   option :stop, aliases: "-s", type: :boolean, default: false, desc: "Stop the stale containers found"
   def stale_containers
-    mutating do
+    stop = options[:stop]
-      stop = options[:stop]
-
-      cli = self
 
+    with_lock_if_stopping do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
-          cli.send(:stale_versions, host: host, role: role).each do |version|
+          app = KAMAL.app(role: role, host: host)
+          versions = capture_with_info(*app.list_versions, raise_on_non_zero_exit: false).split("\n")
+          versions -= [ capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip ]
+
+          versions.each do |version|
             if stop
               puts_by_host host, "Stopping stale container for role #{role} with version #{version}"
-              execute *KAMAL.app(role: role).stop(version: version), raise_on_non_zero_exit: false
+              execute *app.stop(version: version), raise_on_non_zero_exit: false
             else
               puts_by_host host,  "Detected stale container for role #{role} with version #{version} (use `kamal app stale_containers --stop` to stop)"
             end
@@ -197,24 +186,29 @@ class Kamal::Cli::App < Kamal::Cli::Base
   option :since, aliases: "-s", desc: "Show lines since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
   option :lines, type: :numeric, aliases: "-n", desc: "Number of lines to show from each server"
   option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
+  option :grep_options, aliases: "-o", desc: "Additional options supplied to grep"
   option :follow, aliases: "-f", desc: "Follow log on primary server (or specific host set by --hosts)"
   def logs
     # FIXME: Catch when app containers aren't running
 
     grep = options[:grep]
+    grep_options = options[:grep_options]
+    since = options[:since]
 
     if options[:follow]
+      lines = options[:lines].presence || ((since || grep) ? nil : 10) # Default to 10 lines if since or grep isn't set
+
       run_locally do
         info "Following logs on #{KAMAL.primary_host}..."
 
-        KAMAL.specific_roles ||= ["web"]
+        KAMAL.specific_roles ||= [ "web" ]
         role = KAMAL.roles_on(KAMAL.primary_host).first
 
-        info KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, grep: grep)
+        app = KAMAL.app(role: role, host: host)
-        exec KAMAL.app(role: role).follow_logs(host: KAMAL.primary_host, grep: grep)
+        info app.follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep, grep_options: grep_options)
+        exec app.follow_logs(host: KAMAL.primary_host, lines: lines, grep: grep, grep_options: grep_options)
       end
     else
-      since = options[:since]
       lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
       on(KAMAL.hosts) do |host|
@@ -222,7 +216,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
         roles.each do |role|
           begin
-            puts_by_host host, capture_with_info(*KAMAL.app(role: role).logs(since: since, lines: lines, grep: grep))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).logs(since: since, lines: lines, grep: grep, grep_options: grep_options))
           rescue SSHKit::Command::Failed
             puts_by_host host, "Nothing found"
           end
@@ -233,7 +227,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove", "Remove app containers and images from servers"
   def remove
-    mutating do
+    with_lock do
       stop
       remove_containers
       remove_images
@@ -242,13 +236,13 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true
   def remove_container(version)
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
           execute *KAMAL.auditor.record("Removed app container with version #{version}", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).remove_container(version: version)
+          execute *KAMAL.app(role: role, host: host).remove_container(version: version)
         end
       end
     end
@@ -256,13 +250,13 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove_containers", "Remove all app containers from servers", hide: true
   def remove_containers
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do |host|
         roles = KAMAL.roles_on(host)
 
         roles.each do |role|
           execute *KAMAL.auditor.record("Removed all app containers", role: role), verbosity: :debug
-          execute *KAMAL.app(role: role).remove_containers
+          execute *KAMAL.app(role: role, host: host).remove_containers
         end
       end
     end
@@ -270,7 +264,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
   desc "remove_images", "Remove all app images from servers", hide: true
   def remove_images
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Removed all app images"), verbosity: :debug
         execute *KAMAL.app.remove_images
@@ -282,7 +276,7 @@ class Kamal::Cli::App < Kamal::Cli::Base
   def version
     on(KAMAL.hosts) do |host|
       role = KAMAL.roles_on(host).first
-      puts_by_host host, capture_with_info(*KAMAL.app(role: role).current_running_version).strip
+      puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).current_running_version).strip
     end
   end
 
@@ -305,23 +299,20 @@ class Kamal::Cli::App < Kamal::Cli::Base
       version = nil
       on(host) do
         role = KAMAL.roles_on(host).first
-        version = capture_with_info(*KAMAL.app(role: role).current_running_version).strip
+        version = capture_with_info(*KAMAL.app(role: role, host: host).current_running_version).strip
       end
       version.presence
     end
 
-    def stale_versions(host:, role:)
+    def version_or_latest
-      versions = nil
+      options[:version] || KAMAL.config.latest_tag
-      on(host) do
-        versions = \
-          capture_with_info(*KAMAL.app(role: role).list_versions, raise_on_non_zero_exit: false)
-          .split("\n")
-          .drop(1)
-      end
-      versions
     end
 
-    def version_or_latest
+    def with_lock_if_stopping
-      options[:version] || "latest"
+      if options[:stop]
+        with_lock { yield }
+      else
+        yield
+      end
     end
 end

lib/kamal/cli/app/boot.rb

@@ -0,0 +1,141 @@
+class Kamal::Cli::App::Boot
+  attr_reader :host, :role, :version, :barrier, :sshkit
+  delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, :upload!, to: :sshkit
+  delegate :uses_cord?, :assets?, :running_traefik?, to: :role
+
+  def initialize(host, role, sshkit, version, barrier)
+    @host = host
+    @role = role
+    @version = version
+    @barrier = barrier
+    @sshkit = sshkit
+  end
+
+  def run
+    old_version = old_version_renamed_if_clashing
+
+    wait_at_barrier if queuer?
+
+    begin
+      start_new_version
+    rescue => e
+      close_barrier if gatekeeper?
+      stop_new_version
+      raise
+    end
+
+    release_barrier if gatekeeper?
+
+    if old_version
+      stop_old_version(old_version)
+    end
+  end
+
+  private
+    def old_version_renamed_if_clashing
+      if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present?
+        renamed_version = "#{version}_replaced_#{SecureRandom.hex(8)}"
+        info "Renaming container #{version} to #{renamed_version} as already deployed on #{host}"
+        audit("Renaming container #{version} to #{renamed_version}")
+        execute *app.rename_container(version: version, new_version: renamed_version)
+      end
+
+      capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip.presence
+    end
+
+    def start_new_version
+      audit "Booted app version #{version}"
+      hostname = "#{host.to_s[0...51].gsub(/\.+$/, '')}-#{SecureRandom.hex(6)}"
+
+      execute *app.ensure_env_directory
+      upload! role.secrets_io(host), role.secrets_path, mode: "0600"
+
+      if proxy_host?
+        execute *app.run_for_proxy(hostname: hostname)
+        if running_traefik?
+          endpoint = capture_with_info(*app.container_id_for_version(version)).strip
+          raise Kamal::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty?
+          execute *KAMAL.proxy.deploy(role.container_prefix, target: endpoint)
+        else
+          Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+        end
+      else
+        execute *app.tie_cord(role.cord_host_file) if uses_cord?
+        execute *app.run(hostname: hostname)
+        Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+      end
+    end
+
+    def stop_new_version
+      execute *app.stop(version: version), raise_on_non_zero_exit: false
+    end
+
+    def stop_old_version(version)
+      if uses_cord? && !proxy_host?
+        cord = capture_with_info(*app.cord(version: version), raise_on_non_zero_exit: false).strip
+        if cord.present?
+          execute *app.cut_cord(cord)
+          Kamal::Cli::Healthcheck::Poller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+        end
+      end
+
+      execute *app.stop(version: version), raise_on_non_zero_exit: false
+
+      execute *app.clean_up_assets if assets?
+    end
+
+    def release_barrier
+      if barrier.open
+        info "First #{KAMAL.primary_role} container is healthy on #{host}, booting any other roles"
+      end
+    end
+
+    def wait_at_barrier
+      info "Waiting for the first healthy #{KAMAL.primary_role} container before booting #{role} on #{host}..."
+      barrier.wait
+      info "First #{KAMAL.primary_role} container is healthy, booting #{role} on #{host}..."
+    rescue Kamal::Cli::Healthcheck::Error
+      info "First #{KAMAL.primary_role} container is unhealthy, not booting #{role} on #{host}"
+      raise
+    end
+
+    def close_barrier
+      if barrier.close
+        info "First #{KAMAL.primary_role} container is unhealthy on #{host}, not booting any other roles"
+        begin
+          error capture_with_info(*app.logs(version: version))
+          error capture_with_info(*app.container_health_log(version: version))
+        rescue SSHKit::Command::Failed
+          error "Could not fetch logs for #{version}"
+        end
+      end
+    end
+
+    def barrier_role?
+      role == KAMAL.primary_role
+    end
+
+    def app
+      @app ||= KAMAL.app(role: role, host: host)
+    end
+
+    def auditor
+      @auditor = KAMAL.auditor(role: role)
+    end
+
+    def audit(message)
+      execute *auditor.record(message), verbosity: :debug
+    end
+
+    def gatekeeper?
+      barrier && barrier_role?
+    end
+
+    def queuer?
+      barrier && !barrier_role?
+    end
+
+    def proxy_host?
+      KAMAL.proxy_host?(host)
+    end
+end

lib/kamal/cli/app/prepare_assets.rb

@@ -0,0 +1,24 @@
+class Kamal::Cli::App::PrepareAssets
+  attr_reader :host, :role, :sshkit
+  delegate :execute, :capture_with_info, :info, to: :sshkit
+  delegate :assets?, to: :role
+
+  def initialize(host, role, sshkit)
+    @host = host
+    @role = role
+    @sshkit = sshkit
+  end
+
+  def run
+    if assets?
+      execute *app.extract_assets
+      old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+      execute *app.sync_asset_volumes(old_version: old_version)
+    end
+  end
+
+  private
+    def app
+      @app ||= KAMAL.app(role: role, host: host)
+    end
+end

lib/kamal/cli/base.rb

@@ -1,12 +1,12 @@
 require "thor"
-require "dotenv"
 require "kamal/sshkit_with_ext"
 
 module Kamal::Cli
   class Base < Thor
     include SSHKit::DSL
 
-    def self.exit_on_failure?() true end
+    def self.exit_on_failure?() false end
+    def self.dynamic_command_class() Kamal::Cli::Alias::Command end
 
     class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging"
     class_option :quiet, type: :boolean, aliases: "-q", desc: "Minimal logging"
@@ -14,41 +14,31 @@ module Kamal::Cli
     class_option :version, desc: "Run commands against a specific app version"
 
     class_option :primary, type: :boolean, aliases: "-p", desc: "Run commands only on primary host instead of all"
-    class_option :hosts, aliases: "-h", desc: "Run commands on these hosts instead of all (separate by comma)"
+    class_option :hosts, aliases: "-h", desc: "Run commands on these hosts instead of all (separate by comma, supports wildcards with *)"
-    class_option :roles, aliases: "-r", desc: "Run commands on these roles instead of all (separate by comma)"
+    class_option :roles, aliases: "-r", desc: "Run commands on these roles instead of all (separate by comma, supports wildcards with *)"
 
     class_option :config_file, aliases: "-c", default: "config/deploy.yml", desc: "Path to config file"
     class_option :destination, aliases: "-d", desc: "Specify destination to be used for config file (staging -> deploy.staging.yml)"
 
     class_option :skip_hooks, aliases: "-H", type: :boolean, default: false, desc: "Don't run hooks"
 
-    def initialize(*)
+    def initialize(args = [], local_options = {}, config = {})
-      super
+      if config[:current_command].is_a?(Kamal::Cli::Alias::Command)
-      @original_env = ENV.to_h.dup
+        # When Thor generates a dynamic command, it doesn't attempt to parse the arguments.
-      load_envs
+        # For our purposes, it means the arguments are passed in args rather than local_options.
-      initialize_commander(options_with_subcommand_class_options)
+        super([], args, config)
+      else
+        super
+      end
+      initialize_commander unless KAMAL.configured?
     end
 
     private
-      def load_envs
-        if destination = options[:destination]
-          Dotenv.load(".env.#{destination}", ".env")
-        else
-          Dotenv.load(".env")
-        end
-      end
-
-      def reload_envs
-        ENV.clear
-        ENV.update(@original_env)
-        load_envs
-      end
-
       def options_with_subcommand_class_options
         options.merge(@_initializer.last[:class_options] || {})
       end
 
-      def initialize_commander(options)
+      def initialize_commander
         KAMAL.tap do |commander|
           if options[:verbose]
             ENV["VERBOSE"] = "1" # For backtraces via cli/start
@@ -73,37 +63,46 @@ module Kamal::Cli
       def print_runtime
         started_at = Time.now
         yield
-        return Time.now - started_at
+        Time.now - started_at
       ensure
         runtime = Time.now - started_at
         puts "  Finished all in #{sprintf("%.1f seconds", runtime)}"
       end
 
-      def mutating
+      def with_lock
-        return yield if KAMAL.holding_lock?
+        if KAMAL.holding_lock?
-
-        run_hook "pre-connect"
-
-        ensure_run_directory
-
-        acquire_lock
-
-        begin
           yield
-        rescue
+        else
-          if KAMAL.hold_lock_on_error?
+          acquire_lock
-            error "  \e[31mDeploy lock was not released\e[0m"
+
-          else
+          begin
-            release_lock
+            yield
+          rescue
+            begin
+              release_lock
+            rescue => e
+              say "Error releasing the deploy lock: #{e.message}", :red
+            end
+            raise
           end
 
-          raise
+          release_lock
         end
+      end
 
-        release_lock
+      def confirming(question)
+        return yield if options[:confirmed]
+
+        if ask(question, limited_to: %w[ y N ], default: "N") == "y"
+          yield
+        else
+          say "Aborted", :red
+        end
       end
 
       def acquire_lock
+        ensure_run_and_locks_directory
+
         raise_if_locked do
           say "Acquiring the deploy lock...", :magenta
           on(KAMAL.primary_host) { execute *KAMAL.lock.acquire("Automatic deploy lock", KAMAL.config.version), verbosity: :debug }
@@ -123,36 +122,36 @@ module Kamal::Cli
         yield
       rescue SSHKit::Runner::ExecuteError => e
         if e.message =~ /cannot create directory/
+          say "Deploy lock already in place!", :red
           on(KAMAL.primary_host) { puts capture_with_debug(*KAMAL.lock.status) }
-          raise LockError, "Deploy lock found"
+          raise LockError, "Deploy lock found. Run 'kamal lock help' for more information"
         else
           raise e
         end
       end
 
-      def hold_lock_on_error
-        if KAMAL.hold_lock_on_error?
-          yield
-        else
-          KAMAL.hold_lock_on_error = true
-          yield
-          KAMAL.hold_lock_on_error = false
-        end
-      end
-
       def run_hook(hook, **extra_details)
         if !options[:skip_hooks] && KAMAL.hook.hook_exists?(hook)
           details = { hosts: KAMAL.hosts.join(","), command: command, subcommand: subcommand }
 
           say "Running the #{hook} hook...", :magenta
           run_locally do
-            KAMAL.with_verbosity(:debug) { execute *KAMAL.hook.run(hook, **details, **extra_details) }
+            execute *KAMAL.hook.run(hook, **details, **extra_details)
-          rescue SSHKit::Command::Failed
+          rescue SSHKit::Command::Failed => e
-            raise HookError.new("Hook `#{hook}` failed")
+            raise HookError.new("Hook `#{hook}` failed:\n#{e.message}")
           end
         end
       end
 
+      def on(*args, &block)
+        if !KAMAL.connected?
+          run_hook "pre-connect"
+          KAMAL.connected = true
+        end
+
+        super
+      end
+
       def command
         @kamal_command ||= begin
           invocation_class, invocation_commands = *first_invocation
@@ -175,10 +174,14 @@ module Kamal::Cli
         instance_variable_get("@_invocations").first
       end
 
-      def ensure_run_directory
+      def ensure_run_and_locks_directory
         on(KAMAL.hosts) do
           execute(*KAMAL.server.ensure_run_directory)
         end
+
+        on(KAMAL.primary_host) do
+          execute(*KAMAL.lock.ensure_locks_directory)
+        end
       end
-    end
+  end
 end

lib/kamal/cli/build.rb

@@ -5,74 +5,86 @@ class Kamal::Cli::Build < Kamal::Cli::Base
 
   desc "deliver", "Build app and push app image to registry then pull image on servers"
   def deliver
-    mutating do
+    push
-      push
+    pull
-      pull
-    end
   end
 
   desc "push", "Build and push app image to registry"
   def push
-    mutating do
+    cli = self
-      cli = self
 
-      verify_local_dependencies
+    verify_local_dependencies
-      run_hook "pre-build"
+    run_hook "pre-build"
 
-      if (uncommitted_changes = Kamal::Git.uncommitted_changes).present?
+    uncommitted_changes = Kamal::Git.uncommitted_changes
-        say "The following paths have uncommitted changes:\n #{uncommitted_changes}", :yellow
+
+    if KAMAL.config.builder.git_clone?
+      if uncommitted_changes.present?
+        say "Building from a local git clone, so ignoring these uncommitted changes:\n #{uncommitted_changes}", :yellow
       end
 
       run_locally do
-        begin
+        Clone.new(self).prepare
-          KAMAL.with_verbosity(:debug) do
+      end
-            execute *KAMAL.builder.push
+    elsif uncommitted_changes.present?
-          end
+      say "Building with uncommitted changes:\n #{uncommitted_changes}", :yellow
-        rescue SSHKit::Command::Failed => e
+    end
-          if e.message =~ /(no builder)|(no such file or directory)/
-            error "Missing compatible builder, so creating a new one first"
 
-            if cli.create
+    run_locally do
-              KAMAL.with_verbosity(:debug) { execute *KAMAL.builder.push }
+      begin
-            end
+        execute *KAMAL.builder.inspect_builder
-          else
+      rescue SSHKit::Command::Failed => e
-            raise
+        if e.message =~ /(context not found|no builder|no compatible builder|does not exist)/
+          warn "Missing compatible builder, so creating a new one first"
+          begin
+            cli.remove
+          rescue SSHKit::Command::Failed
+            raise unless e.message =~ /(context not found|no builder|does not exist)/
           end
+          cli.create
+        else
+          raise
         end
       end
+
+      # Get the command here to ensure the Dir.chdir doesn't interfere with it
+      push = KAMAL.builder.push
+
+      KAMAL.with_verbosity(:debug) do
+        Dir.chdir(KAMAL.config.builder.build_directory) { execute *push, env: KAMAL.config.builder.secrets }
+      end
     end
   end
 
   desc "pull", "Pull app image from registry onto servers"
   def pull
-    mutating do
+    if (first_hosts = mirror_hosts).any?
-      on(KAMAL.hosts) do
+      #  Pull on a single host per mirror first to seed them
-        execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
+      say "Pulling image on #{first_hosts.join(", ")} to seed the #{"mirror".pluralize(first_hosts.count)}...", :magenta
-        execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
+      pull_on_hosts(first_hosts)
-        execute *KAMAL.builder.pull
+      say "Pulling image on remaining hosts...", :magenta
-        execute *KAMAL.builder.validate_image
+      pull_on_hosts(KAMAL.hosts - first_hosts)
-      end
+    else
+      pull_on_hosts(KAMAL.hosts)
     end
   end
 
   desc "create", "Create a build setup"
   def create
-    mutating do
+    if (remote_host = KAMAL.config.builder.remote)
-      if (remote_host = KAMAL.config.builder.remote_host)
+      connect_to_remote_host(remote_host)
-        connect_to_remote_host(remote_host)
+    end
-      end
 
-      run_locally do
+    run_locally do
-        begin
+      begin
-          debug "Using builder: #{KAMAL.builder.name}"
+        debug "Using builder: #{KAMAL.builder.name}"
-          execute *KAMAL.builder.create
+        execute *KAMAL.builder.create
-        rescue SSHKit::Command::Failed => e
+      rescue SSHKit::Command::Failed => e
-          if e.message =~ /stderr=(.*)/
+        if e.message =~ /stderr=(.*)/
-            error "Couldn't create remote builder: #{$1}"
+          error "Couldn't create remote builder: #{$1}"
-            false
+          false
-          else
+        else
-            raise
+          raise
-          end
         end
       end
     end
@@ -80,11 +92,9 @@ class Kamal::Cli::Build < Kamal::Cli::Base
 
   desc "remove", "Remove build setup"
   def remove
-    mutating do
+    run_locally do
-      run_locally do
+      debug "Using builder: #{KAMAL.builder.name}"
-        debug "Using builder: #{KAMAL.builder.name}"
+      execute *KAMAL.builder.remove
-        execute *KAMAL.builder.remove
-      end
     end
   end
 
@@ -114,10 +124,37 @@ class Kamal::Cli::Build < Kamal::Cli::Base
     def connect_to_remote_host(remote_host)
       remote_uri = URI.parse(remote_host)
       if remote_uri.scheme == "ssh"
-        options = { user: remote_uri.user, port: remote_uri.port }.compact
+        host = SSHKit::Host.new(
-        on(remote_uri.host, options) do
+          hostname: remote_uri.host,
+          ssh_options: { user: remote_uri.user, port: remote_uri.port }.compact
+        )
+        on(host, options) do
           execute "true"
         end
       end
     end
+
+    def mirror_hosts
+      if KAMAL.hosts.many?
+        mirror_hosts = Concurrent::Hash.new
+        on(KAMAL.hosts) do |host|
+          first_mirror = capture_with_info(*KAMAL.builder.first_mirror).strip.presence
+          mirror_hosts[first_mirror] ||= host.to_s if first_mirror
+        rescue SSHKit::Command::Failed => e
+          raise unless e.message =~ /error calling index: reflect: slice index out of range/
+        end
+        mirror_hosts.values
+      else
+        []
+      end
+    end
+
+    def pull_on_hosts(hosts)
+      on(hosts) do
+        execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
+        execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
+        execute *KAMAL.builder.pull
+        execute *KAMAL.builder.validate_image
+      end
+    end
 end

lib/kamal/cli/build/clone.rb

@@ -0,0 +1,61 @@
+require "uri"
+
+class Kamal::Cli::Build::Clone
+  attr_reader :sshkit
+  delegate :info, :error, :execute, :capture_with_info, to: :sshkit
+
+  def initialize(sshkit)
+    @sshkit = sshkit
+  end
+
+  def prepare
+    begin
+      clone_repo
+    rescue SSHKit::Command::Failed => e
+      if e.message =~ /already exists and is not an empty directory/
+        reset
+      else
+        raise Kamal::Cli::Build::BuildError, "Failed to clone repo: #{e.message}"
+      end
+    end
+
+    validate!
+  rescue Kamal::Cli::Build::BuildError => e
+    error "Error preparing clone: #{e.message}, deleting and retrying..."
+
+    FileUtils.rm_rf KAMAL.config.builder.clone_directory
+    clone_repo
+    validate!
+  end
+
+  private
+    def clone_repo
+      info "Cloning repo into build directory `#{KAMAL.config.builder.build_directory}`..."
+
+      FileUtils.mkdir_p KAMAL.config.builder.clone_directory
+      execute *KAMAL.builder.clone
+    end
+
+    def reset
+      info "Resetting local clone as `#{KAMAL.config.builder.build_directory}` already exists..."
+
+      KAMAL.builder.clone_reset_steps.each { |step| execute *step }
+    rescue SSHKit::Command::Failed => e
+      raise Kamal::Cli::Build::BuildError, "Failed to clone repo: #{e.message}"
+    end
+
+    def validate!
+      status = capture_with_info(*KAMAL.builder.clone_status).strip
+
+      unless status.empty?
+        raise Kamal::Cli::Build::BuildError, "Clone in #{KAMAL.config.builder.build_directory} is dirty, #{status}"
+      end
+
+      revision = capture_with_info(*KAMAL.builder.clone_revision).strip
+      if revision != Kamal::Git.revision
+        raise Kamal::Cli::Build::BuildError, "Clone in #{KAMAL.config.builder.build_directory} is not on the correct revision, expected `#{Kamal::Git.revision}` but got `#{revision}`"
+      end
+    rescue SSHKit::Command::Failed => e
+      raise Kamal::Cli::Build::BuildError, "Failed to validate clone: #{e.message}"
+    end
+end

lib/kamal/cli/env.rb

@@ -1,56 +0,0 @@
-require "tempfile"
-
-class Kamal::Cli::Env < Kamal::Cli::Base
-  desc "push", "Push the env file to the remote hosts"
-  def push
-    mutating do
-      on(KAMAL.hosts) do
-        execute *KAMAL.auditor.record("Pushed env files"), verbosity: :debug
-
-        KAMAL.roles_on(host).each do |role|
-          role_config = KAMAL.config.role(role)
-          execute *KAMAL.app(role: role).make_env_directory
-          upload! StringIO.new(role_config.env_file), role_config.host_env_file_path, mode: 400
-        end
-      end
-
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.traefik.make_env_directory
-        upload! StringIO.new(KAMAL.traefik.env_file), KAMAL.traefik.host_env_file_path, mode: 400
-      end
-
-      on(KAMAL.accessory_hosts) do
-        KAMAL.accessories_on(host).each do |accessory|
-          accessory_config = KAMAL.config.accessory(accessory)
-          execute *KAMAL.accessory(accessory).make_env_directory
-          upload! StringIO.new(accessory_config.env_file), accessory_config.host_env_file_path, mode: 400
-        end
-      end
-    end
-  end
-
-  desc "delete", "Delete the env file from the remote hosts"
-  def delete
-    mutating do
-      on(KAMAL.hosts) do
-        execute *KAMAL.auditor.record("Deleted env files"), verbosity: :debug
-
-        KAMAL.roles_on(host).each do |role|
-          role_config = KAMAL.config.role(role)
-          execute *KAMAL.app(role: role).remove_env_file
-        end
-      end
-
-      on(KAMAL.traefik_hosts) do
-        execute *KAMAL.traefik.remove_env_file
-      end
-
-      on(KAMAL.accessory_hosts) do
-        KAMAL.accessories_on(host).each do |accessory|
-          accessory_config = KAMAL.config.accessory(accessory)
-          execute *KAMAL.accessory(accessory).remove_env_file
-        end
-      end
-    end
-  end
-end

lib/kamal/cli/healthcheck.rb

@@ -1,20 +0,0 @@
-class Kamal::Cli::Healthcheck < Kamal::Cli::Base
-  default_command :perform
-
-  desc "perform", "Health check current app version"
-  def perform
-    on(KAMAL.primary_host) do
-      begin
-        execute *KAMAL.healthcheck.run
-        Poller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
-      rescue Poller::HealthcheckError => e
-        error capture_with_info(*KAMAL.healthcheck.logs)
-        error capture_with_pretty_json(*KAMAL.healthcheck.container_health_log)
-        raise
-      ensure
-        execute *KAMAL.healthcheck.stop, raise_on_non_zero_exit: false
-        execute *KAMAL.healthcheck.remove, raise_on_non_zero_exit: false
-      end
-    end
-  end
-end

lib/kamal/cli/healthcheck/barrier.rb

@@ -0,0 +1,33 @@
+require "concurrent/ivar"
+
+class Kamal::Cli::Healthcheck::Barrier
+  def initialize
+    @ivar = Concurrent::IVar.new
+  end
+
+  def close
+    set(false)
+  end
+
+  def open
+    set(true)
+  end
+
+  def wait
+    unless opened?
+      raise Kamal::Cli::Healthcheck::Error.new("Halted at barrier")
+    end
+  end
+
+  private
+    def opened?
+      @ivar.value
+    end
+
+    def set(value)
+      @ivar.set(value)
+      true
+    rescue Concurrent::MultipleAssignmentError
+      false
+    end
+end

lib/kamal/cli/healthcheck/error.rb

@@ -0,0 +1,2 @@
+class Kamal::Cli::Healthcheck::Error < StandardError
+end

lib/kamal/cli/healthcheck/poller.rb

@@ -3,11 +3,10 @@ module Kamal::Cli::Healthcheck::Poller
 
   TRAEFIK_UPDATE_DELAY = 5
 
-  class HealthcheckError < StandardError; end
 
   def wait_for_healthy(pause_after_ready: false, &block)
     attempt = 1
-    max_attempts = KAMAL.config.healthcheck["max_attempts"]
+    max_attempts = KAMAL.config.healthcheck.max_attempts
 
     begin
       case status = block.call
@@ -16,9 +15,9 @@ module Kamal::Cli::Healthcheck::Poller
       when "running" # No health check configured
         sleep KAMAL.config.readiness_delay if pause_after_ready
       else
-        raise HealthcheckError, "container not ready (#{status})"
+        raise Kamal::Cli::Healthcheck::Error, "container not ready (#{status})"
       end
-    rescue HealthcheckError => e
+    rescue Kamal::Cli::Healthcheck::Error => e
       if attempt <= max_attempts
         info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
         sleep attempt
@@ -34,16 +33,16 @@ module Kamal::Cli::Healthcheck::Poller
 
   def wait_for_unhealthy(pause_after_ready: false, &block)
     attempt = 1
-    max_attempts = KAMAL.config.healthcheck["max_attempts"]
+    max_attempts = KAMAL.config.healthcheck.max_attempts
 
     begin
       case status = block.call
       when "unhealthy"
         sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
       else
-        raise HealthcheckError, "container not unhealthy (#{status})"
+        raise Kamal::Cli::Healthcheck::Error, "container not unhealthy (#{status})"
       end
-    rescue HealthcheckError => e
+    rescue Kamal::Cli::Healthcheck::Error => e
       if attempt <= max_attempts
         info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
         sleep attempt

lib/kamal/cli/lock.rb

@@ -3,7 +3,6 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   def status
     handle_missing_lock do
       on(KAMAL.primary_host) do
-        execute *KAMAL.server.ensure_run_directory
         puts capture_with_debug(*KAMAL.lock.status)
       end
     end
@@ -13,9 +12,10 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   option :message, aliases: "-m", type: :string, desc: "A lock message", required: true
   def acquire
     message = options[:message]
+    ensure_run_and_locks_directory
+
     raise_if_locked do
       on(KAMAL.primary_host) do
-        execute *KAMAL.server.ensure_run_directory
         execute *KAMAL.lock.acquire(message, KAMAL.config.version), verbosity: :debug
       end
       say "Acquired the deploy lock"
@@ -26,7 +26,6 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   def release
     handle_missing_lock do
       on(KAMAL.primary_host) do
-        execute *KAMAL.server.ensure_run_directory
         execute *KAMAL.lock.release, verbosity: :debug
       end
       say "Released the deploy lock"

lib/kamal/cli/main.rb

@@ -1,15 +1,15 @@
 class Kamal::Cli::Main < Kamal::Cli::Base
   desc "setup", "Setup all accessories, push the env, and deploy app to servers"
+  option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def setup
     print_runtime do
-      mutating do
+      with_lock do
+        invoke_options = deploy_options
+
         say "Ensure Docker is installed...", :magenta
-        invoke "kamal:cli:server:bootstrap"
+        invoke "kamal:cli:server:bootstrap", [], invoke_options
 
-        say "Push env files...", :magenta
+        invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options
-        invoke "kamal:cli:env:push"
-
-        invoke "kamal:cli:accessory:boot", [ "all" ]
         deploy
       end
     end
@@ -19,28 +19,30 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def deploy
     runtime = print_runtime do
-      mutating do
+      invoke_options = deploy_options
-        invoke_options = deploy_options
 
-        say "Log into image registry...", :magenta
+      say "Log into image registry...", :magenta
-        invoke "kamal:cli:registry:login", [], invoke_options
+      invoke "kamal:cli:registry:login", [], invoke_options.merge(skip_local: options[:skip_push])
 
-        if options[:skip_push]
+      if options[:skip_push]
-          say "Pull app image...", :magenta
+        say "Pull app image...", :magenta
-          invoke "kamal:cli:build:pull", [], invoke_options
+        invoke "kamal:cli:build:pull", [], invoke_options
+      else
+        say "Build and push app image...", :magenta
+        invoke "kamal:cli:build:deliver", [], invoke_options
+      end
+
+      with_lock do
+        run_hook "pre-deploy", secrets: true
+
+        if KAMAL.config.proxy.enabled?
+          say "Ensure Traefik/kamal-proxy is running...", :magenta
+          invoke "kamal:cli:proxy:boot", [], invoke_options
         else
-          say "Build and push app image...", :magenta
+          say "Ensure Traefik is running...", :magenta
-          invoke "kamal:cli:build:deliver", [], invoke_options
+          invoke "kamal:cli:traefik:boot", [], invoke_options
         end
 
-        run_hook "pre-deploy"
-
-        say "Ensure Traefik is running...", :magenta
-        invoke "kamal:cli:traefik:boot", [], invoke_options
-
-        say "Ensure app can pass healthcheck...", :magenta
-        invoke "kamal:cli:healthcheck:perform", [], invoke_options
-
         say "Detect stale containers...", :magenta
         invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
 
@@ -51,28 +53,25 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       end
     end
 
-    run_hook "post-deploy", runtime: runtime.round
+    run_hook "post-deploy", secrets: true, runtime: runtime.round
   end
 
   desc "redeploy", "Deploy app to servers without bootstrapping servers, starting Traefik, pruning, and registry login"
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def redeploy
     runtime = print_runtime do
-      mutating do
+      invoke_options = deploy_options
-        invoke_options = deploy_options
 
-        if options[:skip_push]
+      if options[:skip_push]
-          say "Pull app image...", :magenta
+        say "Pull app image...", :magenta
-          invoke "kamal:cli:build:pull", [], invoke_options
+        invoke "kamal:cli:build:pull", [], invoke_options
-        else
+      else
-          say "Build and push app image...", :magenta
+        say "Build and push app image...", :magenta
-          invoke "kamal:cli:build:deliver", [], invoke_options
+        invoke "kamal:cli:build:deliver", [], invoke_options
-        end
+      end
 
-        run_hook "pre-deploy"
+      with_lock do
-
+        run_hook "pre-deploy", secrets: true
-        say "Ensure app can pass healthcheck...", :magenta
-        invoke "kamal:cli:healthcheck:perform", [], invoke_options
 
         say "Detect stale containers...", :magenta
         invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
@@ -81,21 +80,21 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       end
     end
 
-    run_hook "post-deploy", runtime: runtime.round
+    run_hook "post-deploy", secrets: true, runtime: runtime.round
   end
 
   desc "rollback [VERSION]", "Rollback app to VERSION"
   def rollback(version)
     rolled_back = false
     runtime = print_runtime do
-      mutating do
+      with_lock do
         invoke_options = deploy_options
 
         KAMAL.config.version = version
         old_version = nil
 
         if container_available?(version)
-          run_hook "pre-deploy"
+          run_hook "pre-deploy", secrets: true
 
           invoke "kamal:cli:app:boot", [], invoke_options.merge(version: version)
           rolled_back = true
@@ -105,12 +104,16 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       end
     end
 
-    run_hook "post-deploy", runtime: runtime.round if rolled_back
+    run_hook "post-deploy", secrets: true, runtime: runtime.round if rolled_back
   end
 
   desc "details", "Show details about all containers"
   def details
-    invoke "kamal:cli:traefik:details"
+    if KAMAL.config.proxy.enabled?
+      invoke "kamal:cli:proxy:details"
+    else
+      invoke "kamal:cli:traefik:details"
+    end
     invoke "kamal:cli:app:details"
     invoke "kamal:cli:accessory:details", [ "all" ]
   end
@@ -129,6 +132,18 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     end
   end
 
+  desc "docs", "Show Kamal documentation for configuration setting"
+  def docs(section = nil)
+    case section
+    when NilClass
+      puts Kamal::Configuration.validation_doc
+    else
+      puts Kamal::Configuration.const_get(section.titlecase.to_sym).validation_doc
+    end
+  rescue NameError
+    puts "No documentation found for #{section}"
+  end
+
   desc "init", "Create config stub in config/deploy.yml and env stub in .env"
   option :bundle, type: :boolean, default: false, desc: "Add Kamal to the Gemfile and create a bin/kamal binstub"
   def init
@@ -142,9 +157,10 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       puts "Created configuration file in config/deploy.yml"
     end
 
-    unless (deploy_file = Pathname.new(File.expand_path(".env"))).exist?
+    unless (secrets_file = Pathname.new(File.expand_path(".kamal/secrets"))).exist?
-      FileUtils.cp_r Pathname.new(File.expand_path("templates/template.env", __dir__)), deploy_file
+      FileUtils.mkdir_p secrets_file.dirname
-      puts "Created .env file"
+      FileUtils.cp_r Pathname.new(File.expand_path("templates/secrets", __dir__)), secrets_file
+      puts "Created .kamal/secrets file"
     end
 
     unless (hooks_dir = Pathname.new(File.expand_path(".kamal/hooks"))).exist?
@@ -169,34 +185,19 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     end
   end
 
-  desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)"
-  option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip .env file push"
-  def envify
-    if destination = options[:destination]
-      env_template_path = ".env.#{destination}.erb"
-      env_path          = ".env.#{destination}"
-    else
-      env_template_path = ".env.erb"
-      env_path          = ".env"
-    end
-
-    File.write(env_path, ERB.new(File.read(env_template_path), trim_mode: "-").result, perm: 0600)
-
-    unless options[:skip_push]
-      reload_envs
-      invoke "kamal:cli:env:push", options
-    end
-  end
-
   desc "remove", "Remove Traefik, app, accessories, and registry session from servers"
   option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def remove
-    mutating do
+    confirming "This will remove all containers and images. Are you sure?" do
-      if options[:confirmed] || ask("This will remove all containers and images. Are you sure?", limited_to: %w( y N ), default: "N") == "y"
+      with_lock do
-        invoke "kamal:cli:traefik:remove", [], options.without(:confirmed)
+        if KAMAL.config.proxy.enabled?
+          invoke "kamal:cli:proxy:remove", [], options.without(:confirmed)
+        else
+          invoke "kamal:cli:traefik:remove", [], options.without(:confirmed)
+        end
         invoke "kamal:cli:app:remove", [], options.without(:confirmed)
         invoke "kamal:cli:accessory:remove", [ "all" ], options
-        invoke "kamal:cli:registry:logout", [], options.without(:confirmed)
+        invoke "kamal:cli:registry:logout", [], options.without(:confirmed).merge(skip_local: true)
       end
     end
   end
@@ -215,21 +216,21 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   desc "build", "Build application image"
   subcommand "build", Kamal::Cli::Build
 
-  desc "env", "Manage environment files"
-  subcommand "env", Kamal::Cli::Env
-
-  desc "healthcheck", "Healthcheck application"
-  subcommand "healthcheck", Kamal::Cli::Healthcheck
-
   desc "lock", "Manage the deploy lock"
   subcommand "lock", Kamal::Cli::Lock
 
+  desc "proxy", "Prune old application images and containers"
+  subcommand "proxy", Kamal::Cli::Proxy
+
   desc "prune", "Prune old application images and containers"
   subcommand "prune", Kamal::Cli::Prune
 
   desc "registry", "Login and -out of the image registry"
   subcommand "registry", Kamal::Cli::Registry
 
+  desc "secrets", "Helpers for extracting secrets"
+  subcommand "secrets", Kamal::Cli::Secrets
+
   desc "server", "Bootstrap servers with curl and Docker"
   subcommand "server", Kamal::Cli::Server
 
@@ -241,11 +242,11 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       begin
         on(KAMAL.hosts) do
           KAMAL.roles_on(host).each do |role|
-            container_id = capture_with_info(*KAMAL.app(role: role).container_id_for_version(version))
+            container_id = capture_with_info(*KAMAL.app(role: role, host: host).container_id_for_version(version))
             raise "Container not found" unless container_id.present?
           end
         end
-      rescue SSHKit::Runner::ExecuteError => e
+      rescue SSHKit::Runner::ExecuteError, SSHKit::Runner::MultipleExecuteError => e
         if e.message =~ /Container not found/
           say "Error looking for container version #{version}: #{e.message}"
           return false

lib/kamal/cli/proxy.rb

@@ -0,0 +1,213 @@
+class Kamal::Cli::Proxy < Kamal::Cli::Base
+  desc "boot", "Boot proxy on servers"
+  def boot
+    raise_unless_kamal_proxy_enabled!
+    with_lock do
+      on(KAMAL.hosts) do |host|
+        execute *KAMAL.docker.create_network
+      rescue SSHKit::Command::Failed => e
+        raise unless e.message.include?("already exists")
+      end
+
+      on(KAMAL.traefik_hosts) do |host|
+        execute *KAMAL.registry.login
+        if KAMAL.proxy_host?(host)
+          execute *KAMAL.proxy.start_or_run
+        else
+          execute *KAMAL.traefik.ensure_env_directory
+          upload! KAMAL.traefik.secrets_io, KAMAL.traefik.secrets_path, mode: "0600"
+          execute *KAMAL.traefik.start_or_run
+        end
+      end
+    end
+  end
+
+  desc "reboot", "Reboot proxy on servers (stop container, remove container, start new container)"
+  option :rolling, type: :boolean, default: false, desc: "Reboot proxy on hosts in sequence, rather than in parallel"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  def reboot
+    raise_unless_kamal_proxy_enabled!
+    confirming "This will cause a brief outage on each host. Are you sure?" do
+      with_lock do
+        host_groups = options[:rolling] ? KAMAL.traefik_hosts : [ KAMAL.traefik_hosts ]
+        host_groups.each do |hosts|
+          host_list = Array(hosts).join(",")
+          run_hook "pre-traefik-reboot", hosts: host_list
+          on(hosts) do |host|
+            execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
+            execute *KAMAL.registry.login
+
+            "Stopping and removing Traefik on #{host}, if running..."
+            execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
+            execute *KAMAL.traefik.remove_container
+
+            "Stopping and removing kamal-proxy on #{host}, if running..."
+            execute *KAMAL.proxy.stop, raise_on_non_zero_exit: false
+            execute *KAMAL.proxy.remove_container
+
+            execute *KAMAL.traefik_or_proxy(host).run
+
+            if KAMAL.proxy_host?(host)
+              KAMAL.roles_on(host).select(&:running_traefik?).each do |role|
+                app = KAMAL.app(role: role, host: host)
+
+                version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+                endpoint = capture_with_info(*app.container_id_for_version(version)).strip
+
+                if endpoint.present?
+                  info "Deploying #{endpoint} for role `#{role}` on #{host}..."
+                  execute *KAMAL.proxy.deploy(role.container_prefix, target: endpoint)
+                end
+              end
+            end
+          end
+          run_hook "post-traefik-reboot", hosts: host_list
+        end
+      end
+    end
+  end
+
+  desc "upgrade", "Upgrade to correct proxy on servers (stop container, remove container, start new container)"
+  option :rolling, type: :boolean, default: false, desc: "Reboot proxy on hosts in sequence, rather than in parallel"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  def upgrade
+    invoke_options = { "version" => KAMAL.config.version }.merge(options)
+
+    raise_unless_kamal_proxy_enabled!
+    confirming "This will cause a brief outage on each host. Are you sure?" do
+      host_groups = options[:rolling] ? KAMAL.hosts : [ KAMAL.hosts ]
+      host_groups.each do |hosts|
+        host_list = Array(hosts).join(",")
+        run_hook "pre-traefik-reboot", hosts: host_list
+        on(hosts) do |host|
+          execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug
+          execute *KAMAL.registry.login
+
+          "Stopping and removing Traefik on #{host}, if running..."
+          execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
+          execute *KAMAL.traefik.remove_container
+
+          "Stopping and removing kamal-proxy on #{host}, if running..."
+          execute *KAMAL.proxy.stop, raise_on_non_zero_exit: false
+          execute *KAMAL.proxy.remove_container
+        end
+
+        invoke "kamal:cli:proxy:boot", [], invoke_options.merge("hosts" => host_list)
+        reset_invocation(Kamal::Cli::Proxy)
+        invoke "kamal:cli:app:boot", [], invoke_options.merge("hosts" => host_list, version: KAMAL.config.latest_tag)
+        reset_invocation(Kamal::Cli::App)
+        invoke "kamal:cli:prune:all", [], invoke_options.merge("hosts" => host_list)
+        reset_invocation(Kamal::Cli::Prune)
+
+        run_hook "post-traefik-reboot", hosts: host_list
+      end
+    end
+  end
+
+  desc "start", "Start existing proxy container on servers"
+  def start
+    raise_unless_kamal_proxy_enabled!
+    with_lock do
+      on(KAMAL.traefik_hosts) do |host|
+        execute *KAMAL.auditor.record("Started proxy"), verbosity: :debug
+        execute *KAMAL.traefik_or_proxy(host).start
+      end
+    end
+  end
+
+  desc "stop", "Stop existing proxy container on servers"
+  def stop
+    raise_unless_kamal_proxy_enabled!
+    with_lock do
+      on(KAMAL.traefik_hosts) do |host|
+        execute *KAMAL.auditor.record("Stopped proxy"), verbosity: :debug
+        execute *KAMAL.traefik_or_proxy(host).stop, raise_on_non_zero_exit: false
+      end
+    end
+  end
+
+  desc "restart", "Restart existing proxy container on servers"
+  def restart
+    raise_unless_kamal_proxy_enabled!
+    with_lock do
+      stop
+      start
+    end
+  end
+
+  desc "details", "Show details about proxy container from servers"
+  def details
+    raise_unless_kamal_proxy_enabled!
+    on(KAMAL.traefik_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.traefik_or_proxy(host).info), type: "Proxy" }
+  end
+
+  desc "logs", "Show log lines from proxy on servers"
+  option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
+  option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server"
+  option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
+  option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
+  def logs
+    raise_unless_kamal_proxy_enabled!
+    grep = options[:grep]
+
+    if options[:follow]
+      run_locally do
+        info "Following logs on #{KAMAL.primary_host}..."
+        info KAMAL.traefik_or_proxy(KAMAL.primary_host).follow_logs(host: KAMAL.primary_host, grep: grep)
+        exec KAMAL.traefik_or_proxy(KAMAL.primary_host).follow_logs(host: KAMAL.primary_host, grep: grep)
+      end
+    else
+      since = options[:since]
+      lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
+
+      on(KAMAL.traefik_hosts) do |host|
+        puts_by_host host, capture(*KAMAL.traefik_or_proxy(host).logs(since: since, lines: lines, grep: grep)), type: "Proxy"
+      end
+    end
+  end
+
+  desc "remove", "Remove proxy container and image from servers"
+  def remove
+    raise_unless_kamal_proxy_enabled!
+    with_lock do
+      stop
+      remove_container
+      remove_image
+    end
+  end
+
+  desc "remove_container", "Remove proxy container from servers", hide: true
+  def remove_container
+    raise_unless_kamal_proxy_enabled!
+    with_lock do
+      on(KAMAL.traefik_hosts) do
+        execute *KAMAL.auditor.record("Removed proxy container"), verbosity: :debug
+        execute *KAMAL.proxy.remove_container
+        execute *KAMAL.traefik.remove_container
+      end
+    end
+  end
+
+  desc "remove_image", "Remove proxy image from servers", hide: true
+  def remove_image
+    raise_unless_kamal_proxy_enabled!
+    with_lock do
+      on(KAMAL.traefik_hosts) do
+        execute *KAMAL.auditor.record("Removed proxy image"), verbosity: :debug
+        execute *KAMAL.proxy.remove_image
+        execute *KAMAL.traefik.remove_image
+      end
+    end
+  end
+
+  private
+    def raise_unless_kamal_proxy_enabled!
+      unless KAMAL.config.proxy.enabled?
+        raise "kamal proxy commands are disabled unless experimental proxy support is enabled. Use `kamal traefik` commands instead."
+      end
+    end
+
+    def reset_invocation(cli_class)
+      instance_variable_get("@_invocations")[cli_class].pop
+    end
+end

lib/kamal/cli/prune.rb

@@ -1,7 +1,7 @@
 class Kamal::Cli::Prune < Kamal::Cli::Base
   desc "all", "Prune unused images and stopped containers"
   def all
-    mutating do
+    with_lock do
       containers
       images
     end
@@ -9,7 +9,7 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
 
   desc "images", "Prune unused images"
   def images
-    mutating do
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pruned images"), verbosity: :debug
         execute *KAMAL.prune.dangling_images
@@ -18,12 +18,16 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
     end
   end
 
-  desc "containers", "Prune all stopped containers, except the last 5"
+  desc "containers", "Prune all stopped containers, except the last n (default 5)"
+  option :retain, type: :numeric, default: nil, desc: "Number of containers to retain"
   def containers
-    mutating do
+    retain = options.fetch(:retain, KAMAL.config.retain_containers)
+    raise "retain must be at least 1" if retain < 1
+
+    with_lock do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug
-        execute *KAMAL.prune.app_containers
+        execute *KAMAL.prune.app_containers(retain: retain)
         execute *KAMAL.prune.healthcheck_containers
       end
     end

lib/kamal/cli/registry.rb

@@ -1,18 +1,17 @@
 class Kamal::Cli::Registry < Kamal::Cli::Base
   desc "login", "Log in to registry locally and remotely"
+  option :skip_local, aliases: "-L", type: :boolean, default: false, desc: "Skip local login"
+  option :skip_remote, aliases: "-R", type: :boolean, default: false, desc: "Skip remote login"
   def login
-    run_locally    { execute *KAMAL.registry.login }
+    run_locally    { execute *KAMAL.registry.login } unless options[:skip_local]
-    on(KAMAL.hosts) { execute *KAMAL.registry.login }
+    on(KAMAL.hosts) { execute *KAMAL.registry.login } unless options[:skip_remote]
-  # FIXME: This rescue needed?
-  rescue ArgumentError => e
-    puts e.message
   end
 
-  desc "logout", "Log out of registry remotely"
+  desc "logout", "Log out of registry locally and remotely"
+  option :skip_local, aliases: "-L", type: :boolean, default: false, desc: "Skip local login"
+  option :skip_remote, aliases: "-R", type: :boolean, default: false, desc: "Skip remote login"
   def logout
-    on(KAMAL.hosts) { execute *KAMAL.registry.logout }
+    run_locally    { execute *KAMAL.registry.logout } unless options[:skip_local]
-  # FIXME: This rescue needed?
+    on(KAMAL.hosts) { execute *KAMAL.registry.logout } unless options[:skip_remote]
-  rescue ArgumentError => e
-    puts e.message
   end
 end

lib/kamal/cli/secrets.rb

@@ -0,0 +1,47 @@
+class Kamal::Cli::Secrets < Kamal::Cli::Base
+  desc "fetch [SECRETS...]", "Fetch secrets from a vault"
+  option :adapter, type: :string, aliases: "-a", required: true, desc: "Which vault adapter to use"
+  option :account, type: :string, required: true, desc: "The account identifier or username"
+  option :from, type: :string, required: false, desc: "A vault or folder to fetch the secrets from"
+  option :inline, type: :boolean, required: false, hidden: true
+  def fetch(*secrets)
+    handle_output(inline: options[:inline]) do
+      results = adapter(options[:adapter]).fetch(secrets, **options.slice(:account, :from).symbolize_keys)
+      JSON.dump(results).shellescape
+    end
+  end
+
+  desc "extract", "Extract a single secret from the results of a fetch call"
+  option :inline, type: :boolean, required: false, hidden: true
+  def extract(name, secrets)
+    handle_output(inline: options[:inline]) do
+      parsed_secrets = JSON.parse(secrets)
+
+      value = parsed_secrets[name] || parsed_secrets.find { |k, v| k.end_with?("/#{name}") }&.last
+
+      raise "Could not find secret #{name}" if value.nil?
+
+      value
+    end
+  end
+
+  private
+    def adapter(adapter)
+      Kamal::Secrets::Adapters.lookup(adapter)
+    end
+
+    def handle_output(inline: nil)
+      yield.tap do |output|
+        puts output unless inline
+      end
+    rescue => e
+      handle_error(e)
+    end
+
+    def handle_error(e)
+      $stderr.puts "  \e[31mERROR (#{e.class}): #{e.message}\e[0m"
+      $stderr.puts e.backtrace if ENV["VERBOSE"]
+
+      exit 1
+    end
+end

lib/kamal/cli/server.rb

@@ -1,23 +1,50 @@
 class Kamal::Cli::Server < Kamal::Cli::Base
+  desc "exec", "Run a custom command on the server (use --help to show options)"
+  option :interactive, type: :boolean, aliases: "-i", default: false, desc: "Run the command interactively (use for console/bash)"
+  def exec(*cmd)
+    cmd = Kamal::Utils.join_commands(cmd)
+    hosts = KAMAL.hosts | KAMAL.accessory_hosts
+
+    case
+    when options[:interactive]
+      host = KAMAL.primary_host
+
+      say "Running '#{cmd}' on #{host} interactively...", :magenta
+
+      run_locally { exec KAMAL.server.run_over_ssh(cmd, host: host) }
+    else
+      say "Running '#{cmd}' on #{hosts.join(', ')}...", :magenta
+
+      on(hosts) do |host|
+        execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{host}"), verbosity: :debug
+        puts_by_host host, capture_with_info(cmd)
+      end
+    end
+  end
+
   desc "bootstrap", "Set up Docker to run Kamal apps"
   def bootstrap
-    missing = []
+    with_lock do
+      missing = []
 
-    on(KAMAL.hosts | KAMAL.accessory_hosts) do |host|
+      on(KAMAL.hosts | KAMAL.accessory_hosts) do |host|
-      unless execute(*KAMAL.docker.installed?, raise_on_non_zero_exit: false)
+        unless execute(*KAMAL.docker.installed?, raise_on_non_zero_exit: false)
-        if execute(*KAMAL.docker.superuser?, raise_on_non_zero_exit: false)
+          if execute(*KAMAL.docker.superuser?, raise_on_non_zero_exit: false)
-          info "Missing Docker on #{host}. Installing…"
+            info "Missing Docker on #{host}. Installing…"
-          execute *KAMAL.docker.install
+            execute *KAMAL.docker.install
-        else
+          else
-          missing << host
+            missing << host
+          end
         end
+
+        execute(*KAMAL.server.ensure_run_directory)
       end
 
-      execute(*KAMAL.server.ensure_run_directory)
+      if missing.any?
-    end
+        raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and either `wget` or `curl`. Install Docker manually: https://docs.docker.com/engine/install/"
+      end
 
-    if missing.any?
+      run_hook "docker-setup"
-      raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/"
     end
   end
 end

lib/kamal/cli/templates/deploy.yml

@@ -18,6 +18,10 @@ registry:
   password:
     - KAMAL_REGISTRY_PASSWORD
 
+# Configure builder setup.
+builder:
+  arch: amd64
+
 # Inject ENV variables into containers (secrets come from .env).
 # Remember to run `kamal env push` after making changes!
 # env:
@@ -30,16 +34,6 @@ registry:
 # ssh:
 #   user: app
 
-# Configure builder setup.
-# builder:
-#   args:
-#     RUBY_VERSION: 3.2.0
-#   secrets:
-#     - GITHUB_TOKEN
-#   remote:
-#     arch: amd64
-#     host: ssh://app@192.168.0.1
-
 # Use accessory services (secrets come from .env).
 # accessories:
 #   db:
@@ -63,7 +57,7 @@ registry:
 #     directories:
 #       - data:/data
 
-# Configure custom arguments for Traefik
+# Configure custom arguments for Traefik. Be sure to reboot traefik when you modify it.
 # traefik:
 #   args:
 #     accesslog: true
@@ -77,9 +71,25 @@ registry:
 # Bridge fingerprinted assets, like JS and CSS, between versions to avoid
 # hitting 404 on in-flight requests. Combines all files from new and old
 # version inside the asset_path.
+#
+# If your app is using the Sprockets gem, ensure it sets `config.assets.manifest`.
+# See https://github.com/basecamp/kamal/issues/626 for details
+#
 # asset_path: /rails/public/assets
 
 # Configure rolling deploys by setting a wait time between batches of restarts.
 # boot:
 #   limit: 10 # Can also specify as a percentage of total hosts, such as "25%"
 #   wait: 2
+
+# Configure the role used to determine the primary_host. This host takes
+# deploy locks, runs health checks during the deploy, and follow logs, etc.
+#
+# Caution: there's no support for role renaming yet, so be careful to cleanup
+#          the previous role on the deployed hosts.
+# primary_role: web
+
+# Controls if we abort when see a role with no hosts. Disabling this may be
+# useful for more complex deploy configurations.
+#
+# allow_empty_roles: false

lib/kamal/cli/templates/sample_hooks/docker-setup.sample

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+# A sample docker-setup hook
+#
+# Sets up a Docker network on defined hosts which can then be used by the application’s containers
+
+hosts = ENV["KAMAL_HOSTS"].split(",")
+
+hosts.each do |ip|
+  destination = "root@#{ip}"
+  puts "Creating a Docker network \"kamal\" on #{destination}"
+  `ssh #{destination} docker network create kamal`
+end

lib/kamal/cli/templates/sample_hooks/post-traefik-reboot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooted Traefik on $KAMAL_HOSTS"

lib/kamal/cli/templates/sample_hooks/pre-build.sample

@@ -32,7 +32,7 @@ fi
 current_branch=$(git branch --show-current)
 
 if [ -z "$current_branch" ]; then
-  echo "No git remote set, aborting..." >&2
+  echo "Not on a git branch, aborting..." >&2
   exit 1
 fi
 

lib/kamal/cli/templates/sample_hooks/pre-traefik-reboot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooting Traefik on $KAMAL_HOSTS..."

lib/kamal/cli/templates/secrets

@@ -0,0 +1,16 @@
+# WARNING: Avoid adding secrets directly to this file
+# If you must, then add `.kamal/secrets*` to your .gitignore file
+
+# Option 1: Read secrets from the environment
+KAMAL_REGISTRY_PASSWORD=$KAMAL_REGISTRY_PASSWORD
+
+# Option 2: Read secrets via a command
+# RAILS_MASTER_KEY=$(cat config/master.key)
+
+# Option 3: Read secrets via kamal secrets helpers
+# These will handle logging in and fetching the secrets in as few calls as possible
+# There are adapters for 1Password, LastPass + Bitwarden
+#
+# SECRETS=$(kamal secrets fetch --adapter 1password --account my-account --from MyVault/MyItem KAMAL_REGISTRY_PASSWORD RAILS_MASTER_KEY)
+# KAMAL_REGISTRY_PASSWORD=$(kamal secrets extract KAMAL_REGISTRY_PASSWORD $SECRETS)
+# RAILS_MASTER_KEY=$(kamal secrets extract RAILS_MASTER_KEY $SECRETS)

lib/kamal/cli/templates/template.env

@@ -1,2 +0,0 @@
-KAMAL_REGISTRY_PASSWORD=change-this
-RAILS_MASTER_KEY=another-env

lib/kamal/cli/traefik.rb

@@ -1,9 +1,12 @@
 class Kamal::Cli::Traefik < Kamal::Cli::Base
   desc "boot", "Boot Traefik on servers"
   def boot
-    mutating do
+    raise_if_kamal_proxy_enabled!
+    with_lock do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.registry.login
+        execute *KAMAL.traefik.ensure_env_directory
+        upload! KAMAL.traefik.secrets_io, KAMAL.traefik.secrets_path, mode: "0600"
         execute *KAMAL.traefik.start_or_run
       end
     end
@@ -11,21 +14,32 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
 
   desc "reboot", "Reboot Traefik on servers (stop container, remove container, start new container)"
   option :rolling, type: :boolean, default: false, desc: "Reboot traefik on hosts in sequence, rather than in parallel"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def reboot
-    mutating do
+    raise_if_kamal_proxy_enabled!
-      on(KAMAL.traefik_hosts, in: options[:rolling] ? :sequence : :parallel) do
+    confirming "This will cause a brief outage on each host. Are you sure?" do
-        execute *KAMAL.auditor.record("Rebooted traefik"), verbosity: :debug
+      with_lock do
-        execute *KAMAL.registry.login
+        host_groups = options[:rolling] ? KAMAL.traefik_hosts : [ KAMAL.traefik_hosts ]
-        execute *KAMAL.traefik.stop
+        host_groups.each do |hosts|
-        execute *KAMAL.traefik.remove_container
+          host_list = Array(hosts).join(",")
-        execute *KAMAL.traefik.run
+          run_hook "pre-traefik-reboot", hosts: host_list
+          on(hosts) do
+            execute *KAMAL.auditor.record("Rebooted traefik"), verbosity: :debug
+            execute *KAMAL.registry.login
+            execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
+            execute *KAMAL.traefik.remove_container
+            execute *KAMAL.traefik.run
+          end
+          run_hook "post-traefik-reboot", hosts: host_list
+        end
       end
     end
   end
 
   desc "start", "Start existing Traefik container on servers"
   def start
-    mutating do
+    raise_if_kamal_proxy_enabled!
+    with_lock do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.auditor.record("Started traefik"), verbosity: :debug
         execute *KAMAL.traefik.start
@@ -35,17 +49,19 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
 
   desc "stop", "Stop existing Traefik container on servers"
   def stop
-    mutating do
+    raise_if_kamal_proxy_enabled!
+    with_lock do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.auditor.record("Stopped traefik"), verbosity: :debug
-        execute *KAMAL.traefik.stop
+        execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false
       end
     end
   end
 
   desc "restart", "Restart existing Traefik container on servers"
   def restart
-    mutating do
+    raise_if_kamal_proxy_enabled!
+    with_lock do
       stop
       start
     end
@@ -53,6 +69,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
 
   desc "details", "Show details about Traefik container from servers"
   def details
+    raise_if_kamal_proxy_enabled!
     on(KAMAL.traefik_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.traefik.info), type: "Traefik" }
   end
 
@@ -60,29 +77,33 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
   option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)"
   option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server"
   option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)"
+  option :grep_options, aliases: "-o", desc: "Additional options supplied to grep"
   option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)"
   def logs
+    raise_if_kamal_proxy_enabled!
     grep = options[:grep]
+    grep_options = options[:grep_options]
 
     if options[:follow]
       run_locally do
         info "Following logs on #{KAMAL.primary_host}..."
-        info KAMAL.traefik.follow_logs(host: KAMAL.primary_host, grep: grep)
+        info KAMAL.traefik.follow_logs(host: KAMAL.primary_host, grep: grep, grep_options: grep_options)
-        exec KAMAL.traefik.follow_logs(host: KAMAL.primary_host, grep: grep)
+        exec KAMAL.traefik.follow_logs(host: KAMAL.primary_host, grep: grep, grep_options: grep_options)
       end
     else
       since = options[:since]
       lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
       on(KAMAL.traefik_hosts) do |host|
-        puts_by_host host, capture(*KAMAL.traefik.logs(since: since, lines: lines, grep: grep)), type: "Traefik"
+        puts_by_host host, capture(*KAMAL.traefik.logs(since: since, lines: lines, grep: grep, grep_options: grep_options)), type: "Traefik"
       end
     end
   end
 
   desc "remove", "Remove Traefik container and image from servers"
   def remove
-    mutating do
+    raise_if_kamal_proxy_enabled!
+    with_lock do
       stop
       remove_container
       remove_image
@@ -91,7 +112,8 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
 
   desc "remove_container", "Remove Traefik container from servers", hide: true
   def remove_container
-    mutating do
+    raise_if_kamal_proxy_enabled!
+    with_lock do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.auditor.record("Removed traefik container"), verbosity: :debug
         execute *KAMAL.traefik.remove_container
@@ -101,11 +123,19 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
 
   desc "remove_image", "Remove Traefik image from servers", hide: true
   def remove_image
-    mutating do
+    raise_if_kamal_proxy_enabled!
+    with_lock do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.auditor.record("Removed traefik image"), verbosity: :debug
         execute *KAMAL.traefik.remove_image
       end
     end
   end
+
+  private
+    def raise_if_kamal_proxy_enabled!
+      if KAMAL.config.proxy.enabled?
+        raise "kamal traefik commands are disabled when experimental proxy support is enabled. Use `kamal proxy` commands instead."
+      end
+    end
 end

lib/kamal/commander.rb

@@ -1,13 +1,16 @@
 require "active_support/core_ext/enumerable"
 require "active_support/core_ext/module/delegation"
+require "active_support/core_ext/object/blank"
 
 class Kamal::Commander
-  attr_accessor :verbosity, :holding_lock, :hold_lock_on_error
+  attr_accessor :verbosity, :holding_lock, :connected
+  delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :traefik_hosts, :proxy_hosts, :proxy_host?, :accessory_hosts, to: :specifics
 
   def initialize
     self.verbosity = :info
     self.holding_lock = false
-    self.hold_lock_on_error = false
+    self.connected = false
+    @specifics = nil
   end
 
   def config
@@ -21,52 +24,47 @@ class Kamal::Commander
     @config, @config_kwargs = nil, kwargs
   end
 
+  def configured?
+    @config || @config_kwargs
+  end
+
   attr_reader :specific_roles, :specific_hosts
 
   def specific_primary!
-    self.specific_hosts = [ config.primary_web_host ]
+    @specifics = nil
+    if specific_roles.present?
+      self.specific_hosts = [ specific_roles.first.primary_host ]
+    else
+      self.specific_hosts = [ config.primary_host ]
+    end
   end
 
   def specific_roles=(role_names)
-    @specific_roles = config.roles.select { |r| role_names.include?(r.name) } if role_names.present?
+    @specifics = nil
+    if role_names.present?
+      @specific_roles = Kamal::Utils.filter_specific_items(role_names, config.roles)
+
+      if @specific_roles.empty?
+        raise ArgumentError, "No --roles match for #{role_names.join(',')}"
+      end
+
+      @specific_roles
+    end
   end
 
   def specific_hosts=(hosts)
-    @specific_hosts = config.all_hosts & hosts if hosts.present?
+    @specifics = nil
-  end
+    if hosts.present?
+      @specific_hosts = Kamal::Utils.filter_specific_items(hosts, config.all_hosts)
 
-  def primary_host
+      if @specific_hosts.empty?
-    specific_hosts&.first || specific_roles&.first&.primary_host || config.primary_web_host
+        raise ArgumentError, "No --hosts match for #{hosts.join(',')}"
-  end
+      end
 
-  def primary_role
+      @specific_hosts
-    roles_on(primary_host).first
-  end
-
-  def roles
-    (specific_roles || config.roles).select do |role|
-      ((specific_hosts || config.all_hosts) & role.hosts).any?
     end
   end
 
-  def hosts
-    (specific_hosts || config.all_hosts).select do |host|
-      (specific_roles || config.roles).flat_map(&:hosts).include?(host)
-    end
-  end
-
-  def roles_on(host)
-    roles.select { |role| role.hosts.include?(host.to_s) }.map(&:name)
-  end
-
-  def traefik_hosts
-    specific_hosts || config.traefik_hosts
-  end
-
-  def accessory_hosts
-    specific_hosts || config.accessories.flat_map(&:hosts)
-  end
-
   def accessory_names
     config.accessories&.collect(&:name) || []
   end
@@ -76,8 +74,8 @@ class Kamal::Commander
   end
 
 
-  def app(role: nil)
+  def app(role: nil, host: nil)
-    Kamal::Commands::App.new(config, role: role)
+    Kamal::Commands::App.new(config, role: role, host: host)
   end
 
   def accessory(name)
@@ -108,6 +106,10 @@ class Kamal::Commander
     @lock ||= Kamal::Commands::Lock.new(config)
   end
 
+  def proxy
+    @proxy ||= Kamal::Commands::Proxy.new(config)
+  end
+
   def prune
     @prune ||= Kamal::Commands::Prune.new(config)
   end
@@ -124,6 +126,14 @@ class Kamal::Commander
     @traefik ||= Kamal::Commands::Traefik.new(config)
   end
 
+  def alias(name)
+    config.aliases[name]
+  end
+
+
+  def traefik_or_proxy(host)
+    proxy_host?(host) ? proxy : traefik
+  end
 
   def with_verbosity(level)
     old_level = self.verbosity
@@ -149,8 +159,8 @@ class Kamal::Commander
     self.holding_lock
   end
 
-  def hold_lock_on_error?
+  def connected?
-    self.hold_lock_on_error
+    self.connected
   end
 
   private
@@ -164,4 +174,8 @@ class Kamal::Commander
       SSHKit.config.command_map[:docker] = "docker" # No need to use /usr/bin/env, just clogs up the logs
       SSHKit.config.output_verbosity = verbosity
     end
+
+    def specifics
+      @specifics ||= Kamal::Commander::Specifics.new(config, specific_hosts, specific_roles)
+    end
 end

lib/kamal/commander/specifics.rb

@@ -0,0 +1,58 @@
+class Kamal::Commander::Specifics
+  attr_reader :primary_host, :primary_role, :hosts, :roles
+  delegate :stable_sort!, to: Kamal::Utils
+
+  def initialize(config, specific_hosts, specific_roles)
+    @config, @specific_hosts, @specific_roles = config, specific_hosts, specific_roles
+
+    @roles, @hosts = specified_roles, specified_hosts
+
+    @primary_host = specific_hosts&.first || primary_specific_role&.primary_host || config.primary_host
+    @primary_role = primary_or_first_role(roles_on(primary_host))
+
+    stable_sort!(roles) { |role| role == primary_role ? 0 : 1 }
+    stable_sort!(hosts) { |host| roles_on(host).any? { |role| role == primary_role } ? 0 : 1 }
+  end
+
+  def roles_on(host)
+    roles.select { |role| role.hosts.include?(host.to_s) }
+  end
+
+  def traefik_hosts
+    config.traefik_hosts & specified_hosts
+  end
+
+  def proxy_hosts
+    config.proxy_hosts
+  end
+
+  def proxy_host?(host)
+    host = host.hostname if host.is_a?(SSHKit::Host)
+    proxy_hosts.include?(host)
+  end
+
+  def accessory_hosts
+    specific_hosts || config.accessories.flat_map(&:hosts)
+  end
+
+  private
+    attr_reader :config, :specific_hosts, :specific_roles
+
+    def primary_specific_role
+      primary_or_first_role(specific_roles) if specific_roles.present?
+    end
+
+    def primary_or_first_role(roles)
+      roles.detect { |role| role == config.primary_role } || roles.first
+    end
+
+    def specified_roles
+      (specific_roles || config.roles) \
+        .select { |role| ((specific_hosts || config.all_hosts) & role.hosts).any? }
+    end
+
+    def specified_hosts
+      (specific_hosts || config.all_hosts) \
+        .select { |host| (specific_roles || config.roles).flat_map(&:hosts).include?(host) }
+    end
+end

lib/kamal/commands/accessory.rb

@@ -1,7 +1,9 @@
 class Kamal::Commands::Accessory < Kamal::Commands::Base
   attr_reader :accessory_config
   delegate :service_name, :image, :hosts, :port, :files, :directories, :cmd,
-           :publish_args, :env_args, :volume_args, :label_args, :option_args, to: :accessory_config
+           :publish_args, :env_args, :volume_args, :label_args, :option_args,
+           :secrets_io, :secrets_path, :env_directory,
+           to: :accessory_config
 
   def initialize(config, name:)
     super(config)
@@ -13,6 +15,7 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
       "--name", service_name,
       "--detach",
       "--restart", "unless-stopped",
+      "--network", "kamal",
       *config.logging_args,
       *publish_args,
       *env_args,
@@ -36,17 +39,17 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
   end
 
 
-  def logs(since: nil, lines: nil, grep: nil)
+  def logs(since: nil, lines: nil, grep: nil, grep_options: nil)
     pipe \
       docker(:logs, service_name, (" --since #{since}" if since), (" --tail #{lines}" if lines), "--timestamps", "2>&1"),
-      ("grep '#{grep}'" if grep)
+      ("grep '#{grep}'#{" #{grep_options}" if grep_options}" if grep)
   end
 
-  def follow_logs(grep: nil)
+  def follow_logs(grep: nil, grep_options: nil)
     run_over_ssh \
       pipe \
         docker(:logs, service_name, "--timestamps", "--tail", "10", "--follow", "2>&1"),
-        (%(grep "#{grep}") if grep)
+        (%(grep "#{grep}"#{" #{grep_options}" if grep_options}) if grep)
   end
 
 
@@ -61,6 +64,7 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     docker :run,
       ("-it" if interactive),
       "--rm",
+      "--network", "kamal",
       *env_args,
       *volume_args,
       image,
@@ -98,12 +102,8 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     docker :image, :rm, "--force", image
   end
 
-  def make_env_directory
+  def ensure_env_directory
-    make_directory accessory_config.host_env_directory
+    make_directory env_directory
-  end
-
-  def remove_env_file
-    [:rm, "-f", accessory_config.host_env_file_path]
   end
 
   private

lib/kamal/commands/app.rb

@@ -3,12 +3,14 @@ class Kamal::Commands::App < Kamal::Commands::Base
 
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 
-  attr_reader :role, :role_config
+  attr_reader :role, :host
 
-  def initialize(config, role: nil)
+  delegate :container_name, to: :role
+
+  def initialize(config, role: nil, host: nil)
     super(config)
     @role = role
-    @role_config = config.role(self.role)
+    @host = host
   end
 
   def run(hostname: nil)
@@ -16,18 +18,37 @@ class Kamal::Commands::App < Kamal::Commands::Base
       "--detach",
       "--restart unless-stopped",
       "--name", container_name,
-      *(["--hostname", hostname] if hostname),
+      *([ "--hostname", hostname ] if hostname),
       "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
       "-e", "KAMAL_VERSION=\"#{config.version}\"",
-      *role_config.env_args,
+      *role.env_args(host),
-      *role_config.health_check_args,
+      *role.health_check_args,
-      *config.logging_args,
+      *role.logging_args,
       *config.volume_args,
-      *role_config.asset_volume_args,
+      *role.asset_volume_args,
-      *role_config.label_args,
+      *role.label_args,
-      *role_config.option_args,
+      *role.option_args,
       config.absolute_image,
-      role_config.cmd
+      role.cmd
+  end
+
+  def run_for_proxy(hostname: nil)
+    docker :run,
+      "--detach",
+      "--restart unless-stopped",
+      "--name", container_name,
+      "--network", "kamal",
+      *([ "--hostname", hostname ] if hostname),
+      "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
+      "-e", "KAMAL_VERSION=\"#{config.version}\"",
+      *role.env_args(host),
+      *role.logging_args,
+      *config.volume_args,
+      *role.asset_volume_args,
+      *role.label_args_for_proxy,
+      *role.option_args,
+      config.absolute_image,
+      role.cmd
   end
 
   def start
@@ -50,7 +71,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
 
 
   def current_running_container_id
-    docker :ps, "--quiet", *filter_args(statuses: ACTIVE_DOCKER_STATUSES), "--latest"
+    current_running_container(format: "--quiet")
   end
 
   def container_id_for_version(version, only_running: false)
@@ -58,36 +79,47 @@ class Kamal::Commands::App < Kamal::Commands::Base
   end
 
   def current_running_version
-    list_versions("--latest", statuses: ACTIVE_DOCKER_STATUSES)
+    pipe \
+      current_running_container(format: "--format '{{.Names}}'"),
+      extract_version_from_name
   end
 
   def list_versions(*docker_args, statuses: nil)
     pipe \
       docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
-      %(while read line; do echo ${line##{role_config.container_prefix}-}; done) # Extract SHA from "service-role-dest-SHA"
+      extract_version_from_name
   end
 
-
+  def ensure_env_directory
-  def make_env_directory
+    make_directory role.env_directory
-    make_directory role_config.host_env_directory
   end
 
-  def remove_env_file
-    [ :rm, "-f", role_config.host_env_file_path ]
-  end
-
-
   private
-    def container_name(version = nil)
+    def latest_image_id
-      [ role_config.container_prefix, version || config.version ].compact.join("-")
+      docker :image, :ls, *argumentize("--filter", "reference=#{config.latest_image}"), "--format", "'{{.ID}}'"
+    end
+
+    def current_running_container(format:)
+      pipe \
+        shell(chain(latest_image_container(format: format), latest_container(format: format))),
+        [ :head, "-1" ]
+    end
+
+    def latest_image_container(format:)
+      latest_container format: format, filters: [ "ancestor=$(#{latest_image_id.join(" ")})" ]
+    end
+
+    def latest_container(format:, filters: nil)
+      docker :ps, "--latest", *format, *filter_args(statuses: ACTIVE_DOCKER_STATUSES), argumentize("--filter", filters)
     end
 
     def filter_args(statuses: nil)
       argumentize "--filter", filters(statuses: statuses)
     end
 
-    def service_role_dest
+    def extract_version_from_name
-      [ config.service, role, config.destination ].compact.join("-")
+      # Extract SHA from "service-role-dest-SHA"
+      %(while read line; do echo ${line##{role.container_prefix}-}; done)
     end
 
     def filters(statuses: nil)

lib/kamal/commands/app/assets.rb

@@ -1,23 +1,23 @@
 module Kamal::Commands::App::Assets
   def extract_assets
-    asset_container = "#{role_config.container_prefix}-assets"
+    asset_container = "#{role.container_prefix}-assets"
 
     combine \
-      make_directory(role_config.asset_extracted_path),
+      make_directory(role.asset_extracted_path),
-      [*docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true"],
+      [ *docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true" ],
-      docker(:run, "--name", asset_container, "--detach", "--rm", config.latest_image, "sleep 1000000"),
+      docker(:run, "--name", asset_container, "--detach", "--rm", config.absolute_image, "sleep 1000000"),
-      docker(:cp, "-L", "#{asset_container}:#{role_config.asset_path}/.", role_config.asset_extracted_path),
+      docker(:cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_path),
       docker(:stop, "-t 1", asset_container),
       by: "&&"
   end
 
   def sync_asset_volumes(old_version: nil)
-    new_extracted_path, new_volume_path = role_config.asset_extracted_path(config.version), role_config.asset_volume.host_path
+    new_extracted_path, new_volume_path = role.asset_extracted_path(config.version), role.asset_volume.host_path
     if old_version.present?
-      old_extracted_path, old_volume_path = role_config.asset_extracted_path(old_version), role_config.asset_volume(old_version).host_path
+      old_extracted_path, old_volume_path = role.asset_extracted_path(old_version), role.asset_volume(old_version).host_path
     end
 
-    commands = [make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path)]
+    commands = [ make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path) ]
 
     if old_version.present?
       commands << copy_contents(new_extracted_path, old_volume_path, continue_on_error: true)
@@ -29,8 +29,8 @@ module Kamal::Commands::App::Assets
 
   def clean_up_assets
     chain \
-      find_and_remove_older_siblings(role_config.asset_extracted_path),
+      find_and_remove_older_siblings(role.asset_extracted_path),
-      find_and_remove_older_siblings(role_config.asset_volume_path)
+      find_and_remove_older_siblings(role.asset_volume_path)
   end
 
   private
@@ -39,13 +39,13 @@ module Kamal::Commands::App::Assets
         :find,
         Pathname.new(path).dirname.to_s,
         "-maxdepth 1",
-        "-name", "'#{role_config.container_prefix}-*'",
+        "-name", "'#{role.container_prefix}-*'",
         "!", "-name", Pathname.new(path).basename.to_s,
         "-exec rm -rf \"{}\" +"
       ]
     end
 
     def copy_contents(source, destination, continue_on_error: false)
-      [ :cp, "-rnT", "#{source}", destination, *("|| true" if continue_on_error)]
+      [ :cp, "-rnT", "#{source}", destination, *("|| true" if continue_on_error) ]
     end
 end

lib/kamal/commands/app/containers.rb

@@ -1,4 +1,6 @@
 module Kamal::Commands::App::Containers
+  DOCKER_HEALTH_LOG_FORMAT    = "'{{json .State.Health}}'"
+
   def list_containers
     docker :container, :ls, "--all", *filter_args
   end
@@ -20,4 +22,10 @@ module Kamal::Commands::App::Containers
   def remove_containers
     docker :container, :prune, "--force", *filter_args
   end
+
+  def container_health_log(version:)
+    pipe \
+      container_id_for(container_name: container_name(version)),
+      xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT))
+  end
 end

lib/kamal/commands/app/cord.rb

@@ -2,7 +2,7 @@ module Kamal::Commands::App::Cord
   def cord(version:)
     pipe \
       docker(:inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", container_name(version)),
-      [:awk, "'$2 == \"#{role_config.cord_volume.container_path}\" {print $1}'"]
+      [ :awk, "'$2 == \"#{role.cord_volume.container_path}\" {print $1}'" ]
   end
 
   def tie_cord(cord)
@@ -12,11 +12,11 @@ module Kamal::Commands::App::Cord
   def cut_cord(cord)
     remove_directory(cord)
   end
-  
+
-  private    
+  private
     def create_empty_file(file)
       chain \
         make_directory_for(file),
-        [:touch, file]
+        [ :touch, file ]
     end
 end

lib/kamal/commands/app/execution.rb

@@ -1,27 +1,29 @@
 module Kamal::Commands::App::Execution
-  def execute_in_existing_container(*command, interactive: false)
+  def execute_in_existing_container(*command, interactive: false, env:)
     docker :exec,
       ("-it" if interactive),
+      *argumentize("--env", env),
       container_name,
       *command
   end
 
-  def execute_in_new_container(*command, interactive: false)
+  def execute_in_new_container(*command, interactive: false, env:)
     docker :run,
       ("-it" if interactive),
       "--rm",
-      *role_config&.env_args,
+      *role&.env_args(host),
+      *argumentize("--env", env),
       *config.volume_args,
-      *role_config&.option_args,
+      *role&.option_args,
       config.absolute_image,
       *command
   end
 
-  def execute_in_existing_container_over_ssh(*command, host:)
+  def execute_in_existing_container_over_ssh(*command,  env:)
-    run_over_ssh execute_in_existing_container(*command, interactive: true), host: host
+    run_over_ssh execute_in_existing_container(*command, interactive: true, env: env), host: host
   end
 
-  def execute_in_new_container_over_ssh(*command, host:)
+  def execute_in_new_container_over_ssh(*command, env:)
-    run_over_ssh execute_in_new_container(*command, interactive: true), host: host
+    run_over_ssh execute_in_new_container(*command, interactive: true, env: env), host: host
   end
 end

lib/kamal/commands/app/images.rb

@@ -7,7 +7,7 @@ module Kamal::Commands::App::Images
     docker :image, :prune, "--all", "--force", *filter_args
   end
 
-  def tag_current_image_as_latest
+  def tag_latest_image
     docker :tag, config.absolute_image, config.latest_image
   end
 end

lib/kamal/commands/app/logging.rb

@@ -1,17 +1,17 @@
 module Kamal::Commands::App::Logging
-  def logs(since: nil, lines: nil, grep: nil)
+  def logs(version: nil, since: nil, lines: nil, grep: nil, grep_options: nil)
     pipe \
-      current_running_container_id,
+      version ? container_id_for_version(version) : current_running_container_id,
       "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
-      ("grep '#{grep}'" if grep)
+      ("grep '#{grep}'#{" #{grep_options}" if grep_options}" if grep)
   end
 
-  def follow_logs(host:, grep: nil)
+  def follow_logs(host:, lines: nil, grep: nil, grep_options: nil)
     run_over_ssh \
       pipe(
         current_running_container_id,
-        "xargs docker logs --timestamps --tail 10 --follow 2>&1",
+        "xargs docker logs --timestamps#{" --tail #{lines}" if lines} --follow 2>&1",
-        (%(grep "#{grep}") if grep)
+        (%(grep "#{grep}"#{" #{grep_options}" if grep_options}) if grep)
       ),
       host: host
   end

lib/kamal/commands/auditor.rb

@@ -8,9 +8,12 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
 
   # Runs remotely
   def record(line, **details)
-    append \
+    combine \
-      [ :echo, audit_tags(**details).except(:version, :service_version).to_s, line ],
+      [ :mkdir, "-p", config.run_directory ],
-      audit_log_file
+      append(
+        [ :echo, audit_tags(**details).except(:version, :service_version, :service).to_s, line ],
+        audit_log_file
+      )
   end
 
   def reveal
@@ -21,7 +24,7 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
     def audit_log_file
       file = [ config.service, config.destination, "audit.log" ].compact.join("-")
 
-      "#{config.run_directory}/#{file}"
+      File.join(config.run_directory, file)
     end
 
     def audit_tags(**details)

lib/kamal/commands/base.rb

@@ -3,7 +3,6 @@ module Kamal::Commands
     delegate :sensitive, :argumentize, to: Kamal::Utils
 
     DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'"
-    DOCKER_HEALTH_LOG_FORMAT    = "'{{json .State.Health}}'"
 
     attr_accessor :config
 
@@ -18,7 +17,7 @@ module Kamal::Commands
         elsif config.ssh.proxy && config.ssh.proxy.is_a?(Net::SSH::Proxy::Command)
           cmd << " -o ProxyCommand='#{config.ssh.proxy.command_line_template}'"
         end
-        cmd << " -t #{config.ssh.user}@#{host} '#{command.join(" ")}'"
+        cmd << " -t #{config.ssh.user}@#{host} -p #{config.ssh.port} '#{command.join(" ").gsub("'", "'\\\\''")}'"
       end
     end
 
@@ -38,6 +37,10 @@ module Kamal::Commands
       [ :rm, "-r", path ]
     end
 
+    def remove_file(path)
+      [ :rm, path ]
+    end
+
     private
       def combine(*commands, by: "&&")
         commands
@@ -62,14 +65,30 @@ module Kamal::Commands
         combine *commands, by: ">"
       end
 
+      def any(*commands)
+        combine *commands, by: "||"
+      end
+
       def xargs(command)
         [ :xargs, command ].flatten
       end
 
+      def shell(command)
+        [ :sh, "-c", "'#{command.flatten.join(" ").gsub("'", "'\\\\''")}'" ]
+      end
+
       def docker(*args)
         args.compact.unshift :docker
       end
 
+      def git(*args, path: nil)
+        [ :git, *([ "-C", path ] if path), *args.compact ]
+      end
+
+      def grep(*args)
+        args.compact.unshift :grep
+      end
+
       def tags(**details)
         Kamal::Tags.from_config(config, **details)
       end

lib/kamal/commands/builder.rb

@@ -1,45 +1,37 @@
 require "active_support/core_ext/string/filters"
 
 class Kamal::Commands::Builder < Kamal::Commands::Base
-  delegate :create, :remove, :push, :clean, :pull, :info, :validate_image, to: :target
+  delegate :create, :remove, :push, :clean, :pull, :info, :inspect_builder, :validate_image, :first_mirror, to: :target
+  delegate :local?, :remote?, to: "config.builder"
+
+  include Clone
 
   def name
     target.class.to_s.remove("Kamal::Commands::Builder::").underscore.inquiry
   end
 
   def target
-    case
+    if remote?
-    when !config.builder.multiarch? && !config.builder.cached?
+      if local?
-      native
+        hybrid
-    when !config.builder.multiarch? && config.builder.cached?
+      else
-      native_cached
+        remote
-    when config.builder.local? && config.builder.remote?
+      end
-      multiarch_remote
-    when config.builder.remote?
-      native_remote
     else
-      multiarch
+      local
     end
   end
 
-  def native
+  def remote
-    @native ||= Kamal::Commands::Builder::Native.new(config)
+    @remote ||= Kamal::Commands::Builder::Remote.new(config)
   end
 
-  def native_cached
+  def local
-    @native ||= Kamal::Commands::Builder::Native::Cached.new(config)
+    @local ||= Kamal::Commands::Builder::Local.new(config)
   end
 
-  def native_remote
+  def hybrid
-    @native ||= Kamal::Commands::Builder::Native::Remote.new(config)
+    @hybrid ||= Kamal::Commands::Builder::Hybrid.new(config)
-  end
-
-  def multiarch
-    @multiarch ||= Kamal::Commands::Builder::Multiarch.new(config)
-  end
-
-  def multiarch_remote
-    @multiarch_remote ||= Kamal::Commands::Builder::Multiarch::Remote.new(config)
   end
 
 

lib/kamal/commands/builder/base.rb

@@ -1,20 +1,43 @@
-
 class Kamal::Commands::Builder::Base < Kamal::Commands::Base
   class BuilderError < StandardError; end
 
+  ENDPOINT_DOCKER_HOST_INSPECT = "'{{.Endpoints.docker.Host}}'"
+
   delegate :argumentize, to: Kamal::Utils
-  delegate :args, :secrets, :dockerfile, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, to: :builder_config
+  delegate \
+    :args, :secrets, :dockerfile, :target, :arches, :local_arches, :remote_arches, :remote,
+    :cache_from, :cache_to, :ssh, :driver, :docker_driver?,
+    to: :builder_config
 
   def clean
     docker :image, :rm, "--force", config.absolute_image
   end
 
+  def push
+    docker :buildx, :build,
+      "--push",
+      *platform_options(arches),
+      *([ "--builder", builder_name ] unless docker_driver?),
+      *build_options,
+      build_context
+  end
+
   def pull
     docker :pull, config.absolute_image
   end
 
+  def info
+    combine \
+      docker(:context, :ls),
+      docker(:buildx, :ls)
+  end
+
+  def inspect_builder
+    docker :buildx, :inspect, builder_name unless docker_driver?
+  end
+
   def build_options
-    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile ]
+    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_target, *build_ssh ]
   end
 
   def build_context
@@ -24,9 +47,15 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
   def validate_image
     pipe \
       docker(:inspect, "-f", "'{{ .Config.Labels.service }}'", config.absolute_image),
-      [:grep, "-x", config.service, "||", "(echo \"Image #{config.absolute_image} is missing the `service` label\" && exit 1)"]
+      any(
+        [ :grep, "-x", config.service ],
+        "(echo \"Image #{config.absolute_image} is missing the 'service' label\" && exit 1)"
+      )
   end
 
+  def first_mirror
+    docker(:info, "--format '{{index .RegistryConfig.Mirrors 0}}'")
+  end
 
   private
     def build_tags
@@ -35,8 +64,8 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
 
     def build_cache
       if cache_to && cache_from
-        ["--cache-to", cache_to,
+        [ "--cache-to", cache_to,
-          "--cache-from", cache_from]
+          "--cache-from", cache_from ]
       end
     end
 
@@ -49,7 +78,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
     end
 
     def build_secrets
-      argumentize "--secret", secrets.collect { |secret| [ "id", secret ] }
+      argumentize "--secret", secrets.keys.collect { |secret| [ "id", secret ] }
     end
 
     def build_dockerfile
@@ -60,7 +89,19 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
       end
     end
 
+    def build_target
+      argumentize "--target", target if target.present?
+    end
+
+    def build_ssh
+      argumentize "--ssh", ssh if ssh.present?
+    end
+
     def builder_config
       config.builder
     end
+
+    def platform_options(arches)
+      argumentize "--platform", arches.map { |arch| "linux/#{arch}" }.join(",") if arches.any?
+    end
 end

lib/kamal/commands/builder/clone.rb

@@ -0,0 +1,29 @@
+module Kamal::Commands::Builder::Clone
+  extend ActiveSupport::Concern
+
+  included do
+    delegate :clone_directory, :build_directory, to: :"config.builder"
+  end
+
+  def clone
+    git :clone, Kamal::Git.root, "--recurse-submodules", path: clone_directory
+  end
+
+  def clone_reset_steps
+    [
+      git(:remote, "set-url", :origin, Kamal::Git.root, path: build_directory),
+      git(:fetch, :origin, path: build_directory),
+      git(:reset, "--hard", Kamal::Git.revision, path: build_directory),
+      git(:clean, "-fdx", path: build_directory),
+      git(:submodule, :update, "--init", path: build_directory)
+    ]
+  end
+
+  def clone_status
+    git :status, "--porcelain", path: build_directory
+  end
+
+  def clone_revision
+    git :"rev-parse", :HEAD, path: build_directory
+  end
+end

lib/kamal/commands/builder/hybrid.rb

@@ -0,0 +1,21 @@
+class Kamal::Commands::Builder::Hybrid < Kamal::Commands::Builder::Remote
+  def create
+    combine \
+      create_local_buildx,
+      create_remote_context,
+      append_remote_buildx
+  end
+
+  private
+    def builder_name
+      "kamal-hybrid-#{driver}-#{remote.gsub(/[^a-z0-9_-]/, "-")}"
+    end
+
+    def create_local_buildx
+      docker :buildx, :create, *platform_options(local_arches), "--name", builder_name, "--driver=#{driver}"
+    end
+
+    def append_remote_buildx
+      docker :buildx, :create, *platform_options(remote_arches), "--append", "--name", builder_name, remote_context_name
+    end
+end

lib/kamal/commands/builder/local.rb

@@ -0,0 +1,14 @@
+class Kamal::Commands::Builder::Local < Kamal::Commands::Builder::Base
+  def create
+    docker :buildx, :create, "--name", builder_name, "--driver=#{driver}" unless docker_driver?
+  end
+
+  def remove
+    docker :buildx, :rm, builder_name unless docker_driver?
+  end
+
+  private
+    def builder_name
+      "kamal-local-#{driver}"
+    end
+end

lib/kamal/commands/builder/multiarch.rb

@@ -1,29 +0,0 @@
-class Kamal::Commands::Builder::Multiarch < Kamal::Commands::Builder::Base
-  def create
-    docker :buildx, :create, "--use", "--name", builder_name
-  end
-
-  def remove
-    docker :buildx, :rm, builder_name
-  end
-
-  def push
-    docker :buildx, :build,
-      "--push",
-      "--platform", "linux/amd64,linux/arm64",
-      "--builder", builder_name,
-      *build_options,
-      build_context
-  end
-
-  def info
-    combine \
-      docker(:context, :ls),
-      docker(:buildx, :ls)
-  end
-
-  private
-    def builder_name
-      "kamal-#{config.service}-multiarch"
-    end
-end

lib/kamal/commands/builder/multiarch/remote.rb

@@ -1,51 +0,0 @@
-class Kamal::Commands::Builder::Multiarch::Remote < Kamal::Commands::Builder::Multiarch
-  def create
-    combine \
-      create_contexts,
-      create_local_buildx,
-      append_remote_buildx
-  end
-
-  def remove
-    combine \
-      remove_contexts,
-      super
-  end
-
-  private
-    def builder_name
-      super + "-remote"
-    end
-
-    def builder_name_with_arch(arch)
-      "#{builder_name}-#{arch}"
-    end
-
-    def create_local_buildx
-      docker :buildx, :create, "--name", builder_name, builder_name_with_arch(local_arch), "--platform", "linux/#{local_arch}"
-    end
-
-    def append_remote_buildx
-      docker :buildx, :create, "--append", "--name", builder_name, builder_name_with_arch(remote_arch), "--platform", "linux/#{remote_arch}"
-    end
-
-    def create_contexts
-      combine \
-        create_context(local_arch, local_host),
-        create_context(remote_arch, remote_host)
-    end
-
-    def create_context(arch, host)
-      docker :context, :create, builder_name_with_arch(arch), "--description", "'#{builder_name} #{arch} native host'", "--docker", "'host=#{host}'"
-    end
-
-    def remove_contexts
-      combine \
-        remove_context(local_arch),
-        remove_context(remote_arch)
-    end
-
-    def remove_context(arch)
-      docker :context, :rm, builder_name_with_arch(arch)
-    end
-end

lib/kamal/commands/builder/native.rb

@@ -1,20 +0,0 @@
-class Kamal::Commands::Builder::Native < Kamal::Commands::Builder::Base
-  def create
-    # No-op on native without cache
-  end
-
-  def remove
-    # No-op on native without cache
-  end
-
-  def push
-    combine \
-      docker(:build, *build_options, build_context),
-      docker(:push, config.absolute_image),
-      docker(:push, config.latest_image)
-  end
-
-  def info
-    # No-op on native
-  end
-end

lib/kamal/commands/builder/native/cached.rb

@@ -1,16 +0,0 @@
-class Kamal::Commands::Builder::Native::Cached < Kamal::Commands::Builder::Native
-  def create
-    docker :buildx, :create, "--use", "--driver=docker-container"
-  end
-
-  def remove
-    docker :buildx, :rm, builder_name
-  end
-
-  def push
-    docker :buildx, :build,
-      "--push",
-      *build_options,
-      build_context
-  end
-end

lib/kamal/commands/builder/native/remote.rb

@@ -1,59 +0,0 @@
-class Kamal::Commands::Builder::Native::Remote < Kamal::Commands::Builder::Native
-  def create
-    chain \
-      create_context,
-      create_buildx
-  end
-
-  def remove
-    chain \
-      remove_context,
-      remove_buildx
-  end
-
-  def push
-    docker :buildx, :build,
-      "--push",
-      "--platform", platform,
-      "--builder", builder_name,
-      *build_options,
-      build_context
-  end
-
-  def info
-    chain \
-      docker(:context, :ls),
-      docker(:buildx, :ls)
-  end
-
-
-  private
-    def builder_name
-      "kamal-#{config.service}-native-remote"
-    end
-
-    def builder_name_with_arch
-      "#{builder_name}-#{remote_arch}"
-    end
-
-    def platform
-      "linux/#{remote_arch}"
-    end
-
-    def create_context
-      docker :context, :create,
-        builder_name_with_arch, "--description", "'#{builder_name} #{remote_arch} native host'", "--docker", "'host=#{remote_host}'"
-    end
-
-    def remove_context
-      docker :context, :rm, builder_name_with_arch
-    end
-
-    def create_buildx
-      docker :buildx, :create, "--name", builder_name, builder_name_with_arch, "--platform", platform
-    end
-
-    def remove_buildx
-      docker :buildx, :rm, builder_name
-    end
-end

lib/kamal/commands/builder/remote.rb

@@ -0,0 +1,63 @@
+class Kamal::Commands::Builder::Remote < Kamal::Commands::Builder::Base
+  def create
+    chain \
+      create_remote_context,
+      create_buildx
+  end
+
+  def remove
+    chain \
+      remove_remote_context,
+      remove_buildx
+  end
+
+  def info
+    chain \
+      docker(:context, :ls),
+      docker(:buildx, :ls)
+  end
+
+  def inspect_builder
+    combine \
+      combine inspect_buildx, inspect_remote_context,
+      [ "(echo no compatible builder && exit 1)" ],
+      by: "||"
+  end
+
+  private
+    def builder_name
+      "kamal-remote-#{remote.gsub(/[^a-z0-9_-]/, "-")}"
+    end
+
+    def remote_context_name
+      "#{builder_name}-context"
+    end
+
+    def inspect_buildx
+      pipe \
+        docker(:buildx, :inspect, builder_name),
+        grep("-q", "Endpoint:.*#{remote_context_name}")
+    end
+
+    def inspect_remote_context
+      pipe \
+        docker(:context, :inspect, remote_context_name, "--format", ENDPOINT_DOCKER_HOST_INSPECT),
+        grep("-xq", remote)
+    end
+
+    def create_remote_context
+      docker :context, :create, remote_context_name, "--description", "'#{builder_name} host'", "--docker", "'host=#{remote}'"
+    end
+
+    def remove_remote_context
+      docker :context, :rm, remote_context_name
+    end
+
+    def create_buildx
+      docker :buildx, :create, "--name", builder_name, remote_context_name
+    end
+
+    def remove_buildx
+      docker :buildx, :rm, builder_name
+    end
+end

lib/kamal/commands/docker.rb

@@ -1,7 +1,7 @@
 class Kamal::Commands::Docker < Kamal::Commands::Base
   # Install Docker using the https://github.com/docker/docker-install convenience script.
   def install
-    pipe [ :curl, "-fsSL", "https://get.docker.com" ], :sh
+    pipe get_docker, :sh
   end
 
   # Checks the Docker client version. Fails if Docker is not installed.
@@ -18,4 +18,17 @@ class Kamal::Commands::Docker < Kamal::Commands::Base
   def superuser?
     [ '[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null' ]
   end
+
+  def create_network
+    docker :network, :create, :kamal
+  end
+
+  private
+    def get_docker
+      shell \
+        any \
+          [ :curl, "-fsSL", "https://get.docker.com" ],
+          [ :wget, "-O -", "https://get.docker.com" ],
+          [ :echo, "\"exit 1\"" ]
+    end
 end

lib/kamal/commands/healthcheck.rb

@@ -1,60 +0,0 @@
-class Kamal::Commands::Healthcheck < Kamal::Commands::Base
-
-  def run
-    web = config.role(:web)
-
-    docker :run,
-      "--detach",
-      "--name", container_name_with_version,
-      "--publish", "#{exposed_port}:#{config.healthcheck["port"]}",
-      "--label", "service=#{config.healthcheck_service}",
-      "-e", "KAMAL_CONTAINER_NAME=\"#{config.healthcheck_service}\"",
-      *web.env_args,
-      *web.health_check_args(cord: false),
-      *config.volume_args,
-      *web.option_args,
-      config.absolute_image,
-      web.cmd
-  end
-
-  def status
-    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_STATUS_FORMAT))
-  end
-
-  def container_health_log
-    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT))
-  end
-
-  def logs
-    pipe container_id, xargs(docker(:logs, "--tail", log_lines, "2>&1"))
-  end
-
-  def stop
-    pipe container_id, xargs(docker(:stop))
-  end
-
-  def remove
-    pipe container_id, xargs(docker(:container, :rm))
-  end
-
-  private
-    def container_name_with_version
-      "#{config.healthcheck_service}-#{config.version}"
-    end
-
-    def container_id
-      container_id_for(container_name: container_name_with_version)
-    end
-
-    def health_url
-      "http://localhost:#{exposed_port}#{config.healthcheck["path"]}"
-    end
-
-    def exposed_port
-      config.healthcheck["exposed_port"]
-    end
-
-    def log_lines
-      config.healthcheck["log_lines"]
-    end
-end

lib/kamal/commands/hook.rb

@@ -1,6 +1,9 @@
 class Kamal::Commands::Hook < Kamal::Commands::Base
-  def run(hook, **details)
+  def run(hook, secrets: false, **details)
-    [ hook_file(hook), env: tags(**details).env ]
+    env = tags(**details).env
+    env.merge!(config.secrets.to_h) if secrets
+
+    [ hook_file(hook), env: env ]
   end
 
   def hook_exists?(hook)
@@ -9,6 +12,6 @@ class Kamal::Commands::Hook < Kamal::Commands::Base
 
   private
     def hook_file(hook)
-      "#{config.hooks_path}/#{hook}"
+      File.join(config.hooks_path, hook)
     end
 end

lib/kamal/commands/lock.rb

@@ -1,17 +1,18 @@
 require "active_support/duration"
 require "time"
+require "base64"
 
 class Kamal::Commands::Lock < Kamal::Commands::Base
   def acquire(message, version)
     combine \
-      [:mkdir, lock_dir],
+      [ :mkdir, lock_dir ],
       write_lock_details(message, version)
   end
 
   def release
     combine \
-      [:rm, lock_details_file],
+      [ :rm, lock_details_file ],
-      [:rm, "-r", lock_dir]
+      [ :rm, "-r", lock_dir ]
   end
 
   def status
@@ -20,31 +21,41 @@ class Kamal::Commands::Lock < Kamal::Commands::Base
       read_lock_details
   end
 
+  def ensure_locks_directory
+    [ :mkdir, "-p", locks_dir ]
+  end
+
   private
     def write_lock_details(message, version)
       write \
-        [:echo, "\"#{Base64.encode64(lock_details(message, version))}\""],
+        [ :echo, "\"#{Base64.encode64(lock_details(message, version))}\"" ],
         lock_details_file
     end
 
     def read_lock_details
       pipe \
-        [:cat, lock_details_file],
+        [ :cat, lock_details_file ],
-        [:base64, "-d"]
+        [ :base64, "-d" ]
     end
 
     def stat_lock_dir
       write \
-        [:stat, lock_dir],
+        [ :stat, lock_dir ],
         "/dev/null"
     end
 
+    def locks_dir
+      File.join(config.run_directory, "locks")
+    end
+
     def lock_dir
-      "#{config.run_directory}/lock-#{config.service}"
+      dir_name = [ config.service, config.destination ].compact.join("-")
+
+      File.join(locks_dir, dir_name)
     end
 
     def lock_details_file
-      [lock_dir, :details].join("/")
+      File.join(lock_dir, "details")
     end
 
     def lock_details(message, version)

lib/kamal/commands/proxy.rb

@@ -0,0 +1,69 @@
+class Kamal::Commands::Proxy < Kamal::Commands::Base
+  delegate :argumentize, :optionize, to: Kamal::Utils
+  delegate :container_name, :app_port, to: :proxy_config
+
+  attr_reader :proxy_config
+
+  def initialize(config)
+    super
+    @proxy_config = config.proxy
+  end
+
+  def run
+    docker :run,
+      "--name", container_name,
+      "--network", "kamal",
+      "--detach",
+      "--restart", "unless-stopped",
+      *proxy_config.publish_args,
+      "--volume", "/var/run/docker.sock:/var/run/docker.sock",
+      "--volume", "#{proxy_config.config_directory_as_docker_volume}:/root/.config/kamal-proxy",
+      *config.logging_args,
+      proxy_config.image
+  end
+
+  def start
+    docker :container, :start, container_name
+  end
+
+  def stop(name: container_name)
+    docker :container, :stop, name
+  end
+
+  def start_or_run
+    combine start, run, by: "||"
+  end
+
+  def deploy(service, target:)
+    docker :exec, container_name, "kamal-proxy", :deploy, service, *optionize({ target: "#{target}:#{app_port}" }), *proxy_config.deploy_command_args
+  end
+
+  def remove(service, target:)
+    docker :exec, container_name, "kamal-proxy", :remove, service, *optionize({ target: "#{target}:#{app_port}" })
+  end
+
+  def info
+    docker :ps, "--filter", "name=^#{container_name}$"
+  end
+
+  def logs(since: nil, lines: nil, grep: nil, grep_options: nil)
+    pipe \
+      docker(:logs, container_name, (" --since #{since}" if since), (" --tail #{lines}" if lines), "--timestamps", "2>&1"),
+      ("grep '#{grep}'#{" #{grep_options}" if grep_options}" if grep)
+  end
+
+  def follow_logs(host:, grep: nil, grep_options: nil)
+    run_over_ssh pipe(
+      docker(:logs, container_name, "--timestamps", "--tail", "10", "--follow", "2>&1"),
+      (%(grep "#{grep}"#{" #{grep_options}" if grep_options}) if grep)
+    ).join(" "), host: host
+  end
+
+  def remove_container
+    docker :container, :prune, "--force", "--filter", "label=org.opencontainers.image.title=kamal-proxy"
+  end
+
+  def remove_image
+    docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=kamal-proxy"
+  end
+end

lib/kamal/commands/prune.rb

@@ -9,14 +9,14 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
   def tagged_images
     pipe \
       docker(:image, :ls, *service_filter, "--format", "'{{.ID}} {{.Repository}}:{{.Tag}}'"),
-      "grep -v -w \"#{active_image_list}\"",
+      grep("-v -w \"#{active_image_list}\""),
       "while read image tag; do docker rmi $tag; done"
   end
 
-  def app_containers(keep_last: 5)
+  def app_containers(retain:)
     pipe \
       docker(:ps, "-q", "-a", *service_filter, *stopped_containers_filters),
-      "tail -n +#{keep_last + 1}",
+      "tail -n +#{retain + 1}",
       "while read container_id; do docker rm $container_id; done"
   end
 
@@ -26,7 +26,7 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
 
   private
     def stopped_containers_filters
-      [ "created", "exited", "dead" ].flat_map { |status| ["--filter", "status=#{status}"] }
+      [ "created", "exited", "dead" ].flat_map { |status| [ "--filter", "status=#{status}" ] }
     end
 
     def active_image_list
@@ -43,4 +43,4 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
     def healthcheck_service_filter
       [ "--filter", "label=service=#{config.healthcheck_service}" ]
     end
-  end
+end

lib/kamal/commands/registry.rb

@@ -2,19 +2,13 @@ class Kamal::Commands::Registry < Kamal::Commands::Base
   delegate :registry, to: :config
 
   def login
-    docker :login, registry["server"], "-u", sensitive(lookup("username")), "-p", sensitive(lookup("password"))
+    docker :login,
+      registry.server,
+      "-u", sensitive(Kamal::Utils.escape_shell_value(registry.username)),
+      "-p", sensitive(Kamal::Utils.escape_shell_value(registry.password))
   end
 
   def logout
-    docker :logout, registry["server"]
+    docker :logout, registry.server
   end
-
-  private
-    def lookup(key)
-      if registry[key].is_a?(Array)
-        ENV.fetch(registry[key].first).dup
-      else
-        registry[key]
-      end
-    end
 end

lib/kamal/commands/server.rb

@@ -1,5 +1,5 @@
 class Kamal::Commands::Server < Kamal::Commands::Base
   def ensure_run_directory
-    [:mkdir, "-p", config.run_directory]
+    [ :mkdir, "-p", config.run_directory ]
   end
 end

lib/kamal/commands/traefik.rb

@@ -1,19 +1,6 @@
 class Kamal::Commands::Traefik < Kamal::Commands::Base
   delegate :argumentize, :optionize, to: Kamal::Utils
-
+  delegate :port, :publish?, :labels, :env, :image, :options, :args, :env_args, :secrets_io, :env_directory, :secrets_path, to: :"config.traefik"
-  DEFAULT_IMAGE = "traefik:v2.9"
-  CONTAINER_PORT = 80
-  DEFAULT_ARGS = {
-    'log.level' => 'DEBUG'
-  }
-  DEFAULT_LABELS = {
-    # These ensure we serve a 502 rather than a 404 if no containers are available
-    "traefik.http.routers.catchall.entryPoints" => "http",
-    "traefik.http.routers.catchall.rule" => "PathPrefix(`/`)",
-    "traefik.http.routers.catchall.service" => "unavailable",
-    "traefik.http.routers.catchall.priority" => 1,
-    "traefik.http.services.unavailable.loadbalancer.server.port" => "0"
-  }
 
   def run
     docker :run, "--name traefik",
@@ -39,23 +26,23 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
   end
 
   def start_or_run
-    combine start, run, by: "||"
+    any start, run
   end
 
   def info
     docker :ps, "--filter", "name=^traefik$"
   end
 
-  def logs(since: nil, lines: nil, grep: nil)
+  def logs(since: nil, lines: nil, grep: nil, grep_options: nil)
     pipe \
       docker(:logs, "traefik", (" --since #{since}" if since), (" --tail #{lines}" if lines), "--timestamps", "2>&1"),
-      ("grep '#{grep}'" if grep)
+      ("grep '#{grep}'#{" #{grep_options}" if grep_options}" if grep)
   end
 
-  def follow_logs(host:, grep: nil)
+  def follow_logs(host:, grep: nil, grep_options: nil)
     run_over_ssh pipe(
       docker(:logs, "traefik", "--timestamps", "--tail", "10", "--follow", "2>&1"),
-      (%(grep "#{grep}") if grep)
+      (%(grep "#{grep}"#{" #{grep_options}" if grep_options}) if grep)
     ).join(" "), host: host
   end
 
@@ -67,64 +54,24 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
     docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
   end
 
-  def port
+  def ensure_env_directory
-    "#{host_port}:#{CONTAINER_PORT}"
+    make_directory env_directory
-  end
-
-  def env_file
-    Kamal::EnvFile.new(config.traefik.fetch("env", {}))
-  end
-
-  def host_env_file_path
-    File.join host_env_directory, "traefik.env"
-  end
-
-  def make_env_directory
-    make_directory(host_env_directory)
-  end
-
-  def remove_env_file
-    [:rm, "-f", host_env_file_path]
   end
 
   private
     def publish_args
-      argumentize "--publish", port unless config.traefik["publish"] == false
+      argumentize "--publish", port if publish?
     end
 
     def label_args
       argumentize "--label", labels
     end
 
-    def env_args
-      argumentize "--env-file", host_env_file_path
-    end
-
-    def host_env_directory
-      File.join config.host_env_directory, "traefik"
-    end
-
-    def labels
-      DEFAULT_LABELS.merge(config.traefik["labels"] || {})
-    end
-
-    def image
-      config.traefik.fetch("image") { DEFAULT_IMAGE }
-    end
-
     def docker_options_args
-      optionize(config.traefik["options"] || {})
+      optionize(options)
     end
 
     def cmd_option_args
-      if args = config.traefik["args"]
+      optionize args, with: "="
-        optionize DEFAULT_ARGS.merge(args), with: "="
-      else
-        optionize DEFAULT_ARGS, with: "="
-      end
-    end
-
-    def host_port
-      config.traefik["host_port"] || CONTAINER_PORT
     end
 end

lib/kamal/configuration.rb

@@ -1,15 +1,18 @@
 require "active_support/ordered_options"
 require "active_support/core_ext/string/inquiry"
 require "active_support/core_ext/module/delegation"
-require "pathname"
+require "active_support/core_ext/hash/keys"
 require "erb"
 require "net/ssh/proxy/jump"
 
 class Kamal::Configuration
-  delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
+  delegate :service, :image, :labels, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
   delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_reader :destination, :raw_config
+  attr_reader :accessories, :aliases, :boot, :builder, :env, :healthcheck, :logging, :proxy, :traefik, :servers, :ssh, :sshkit, :registry
+
+  include Validation
 
   class << self
     def create_from(config_file:, destination: nil, version: nil)
@@ -25,7 +28,9 @@ class Kamal::Configuration
 
       def load_config_file(file)
         if file.exist?
-          YAML.load(ERB.new(IO.read(file)).result).symbolize_keys
+          # Newer Psych doesn't load aliases by default
+          load_method = YAML.respond_to?(:unsafe_load) ? :unsafe_load : :load
+          YAML.send(load_method, ERB.new(IO.read(file)).result).symbolize_keys
         else
           raise "Configuration file not found in #{file}"
         end
@@ -40,7 +45,31 @@ class Kamal::Configuration
     @raw_config = ActiveSupport::InheritableOptions.new(raw_config)
     @destination = destination
     @declared_version = version
-    valid? if validate
+
+    validate! raw_config, example: validation_yml.symbolize_keys, context: "", with: Kamal::Configuration::Validator::Configuration
+
+    # Eager load config to validate it, these are first as they have dependencies later on
+    @servers = Servers.new(config: self)
+    @registry = Registry.new(config: self)
+
+    @accessories = @raw_config.accessories&.keys&.collect { |name| Accessory.new(name, config: self) } || []
+    @aliases = @raw_config.aliases&.keys&.to_h { |name| [ name, Alias.new(name, config: self) ] } || {}
+    @boot = Boot.new(config: self)
+    @builder = Builder.new(config: self)
+    @env = Env.new(config: @raw_config.env || {}, secrets: secrets)
+
+    @healthcheck = Healthcheck.new(healthcheck_config: @raw_config.healthcheck)
+    @logging = Logging.new(logging_config: @raw_config.logging)
+    @proxy = Proxy.new(config: self)
+    @traefik = Traefik.new(config: self)
+    @ssh = Ssh.new(config: self)
+    @sshkit = Sshkit.new(config: self)
+
+    ensure_destination_if_required
+    ensure_required_keys_present
+    ensure_valid_kamal_version
+    ensure_retain_containers_valid
+    ensure_valid_service_name
   end
 
 
@@ -69,37 +98,56 @@ class Kamal::Configuration
 
 
   def roles
-    @roles ||= role_names.collect { |role_name| Role.new(role_name, config: self) }
+    servers.roles
   end
 
   def role(name)
     roles.detect { |r| r.name == name.to_s }
   end
 
-  def accessories
-    @accessories ||= raw_config.accessories&.keys&.collect { |name| Kamal::Configuration::Accessory.new(name, config: self) } || []
-  end
-
   def accessory(name)
     accessories.detect { |a| a.name == name.to_s }
   end
 
 
   def all_hosts
-    roles.flat_map(&:hosts).uniq
+    (roles + accessories).flat_map(&:hosts).uniq
   end
 
-  def primary_web_host
+  def primary_host
-    role(:web).primary_host
+    primary_role&.primary_host
+  end
+
+  def primary_role_name
+    raw_config.primary_role || "web"
+  end
+
+  def primary_role
+    role(primary_role_name)
+  end
+
+  def allow_empty_roles?
+    raw_config.allow_empty_roles
+  end
+
+  def traefik_roles
+    roles.select(&:running_traefik?)
+  end
+
+  def traefik_role_names
+    traefik_roles.flat_map(&:name)
   end
 
   def traefik_hosts
-    roles.select(&:running_traefik?).flat_map(&:hosts).uniq
+    traefik_roles.flat_map(&:hosts).uniq
   end
 
+  def proxy_hosts
+    proxy.hosts
+  end
 
   def repository
-    [ raw_config.registry["server"], image ].compact.join("/")
+    [ registry.server, image ].compact.join("/")
   end
 
   def absolute_image
@@ -107,7 +155,11 @@ class Kamal::Configuration
   end
 
   def latest_image
-    "#{repository}:latest"
+    "#{repository}:#{latest_tag}"
+  end
+
+  def latest_tag
+    [ "latest", *destination ].join("-")
   end
 
   def service_with_version
@@ -118,6 +170,10 @@ class Kamal::Configuration
     raw_config.require_destination
   end
 
+  def retain_containers
+    raw_config.retain_containers || 5
+  end
+
 
   def volume_args
     if raw_config.volumes.present?
@@ -128,40 +184,10 @@ class Kamal::Configuration
   end
 
   def logging_args
-    if raw_config.logging.present?
+    logging.args
-      optionize({ "log-driver" => raw_config.logging["driver"] }.compact) +
-        argumentize("--log-opt", raw_config.logging["options"])
-    else
-      argumentize("--log-opt", { "max-size" => "10m" })
-    end
   end
 
 
-  def boot
-    Kamal::Configuration::Boot.new(config: self)
-  end
-
-  def builder
-    Kamal::Configuration::Builder.new(config: self)
-  end
-
-  def traefik
-    raw_config.traefik || {}
-  end
-
-  def ssh
-    Kamal::Configuration::Ssh.new(config: self)
-  end
-
-  def sshkit
-    Kamal::Configuration::Sshkit.new(config: self)
-  end
-
-
-  def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }.merge(raw_config.healthcheck || {})
-  end
-
   def healthcheck_service
     [ "healthcheck", service, destination ].compact.join("-")
   end
@@ -176,39 +202,44 @@ class Kamal::Configuration
 
 
   def run_directory
-    raw_config.run_directory || ".kamal"
+    ".kamal"
   end
 
   def run_directory_as_docker_volume
-    if Pathname.new(run_directory).absolute?
+    File.join "$(pwd)", run_directory
-      run_directory
-    else
-      File.join "$(pwd)", run_directory
-    end
   end
 
   def hooks_path
     raw_config.hooks_path || ".kamal/hooks"
   end
 
-  def host_env_directory
-    "#{run_directory}/env"
-  end
-
   def asset_path
     raw_config.asset_path
   end
 
 
-  def valid?
+  def env_directory
-    ensure_destination_if_required && ensure_required_keys_present && ensure_valid_kamal_version
+    File.join(run_directory, "env")
   end
 
+  def env_tags
+    @env_tags ||= if (tags = raw_config.env["tags"])
+      tags.collect { |name, config| Env::Tag.new(name, config: config, secrets: secrets) }
+    else
+      []
+    end
+  end
+
+  def env_tag(name)
+    env_tags.detect { |t| t.name == name.to_s }
+  end
+
+
   def to_h
     {
       roles: role_names,
       hosts: all_hosts,
-      primary_host: primary_web_host,
+      primary_host: primary_host,
       version: version,
       repository: repository,
       absolute_image: absolute_image,
@@ -219,10 +250,13 @@ class Kamal::Configuration
       builder: builder.to_h,
       accessories: raw_config.accessories,
       logging: logging_args,
-      healthcheck: healthcheck
+      healthcheck: healthcheck.to_h
     }.compact
   end
 
+  def secrets
+    @secrets ||= Kamal::Secrets.new(destination: destination)
+  end
 
   private
     # Will raise ArgumentError if any required config keys are missing
@@ -236,34 +270,48 @@ class Kamal::Configuration
 
     def ensure_required_keys_present
       %i[ service image registry servers ].each do |key|
-        raise ArgumentError, "Missing required configuration for #{key}" unless raw_config[key].present?
+        raise Kamal::ConfigurationError, "Missing required configuration for #{key}" unless raw_config[key].present?
       end
 
-      if raw_config.registry["username"].blank?
+      unless role(primary_role_name).present?
-        raise ArgumentError, "You must specify a username for the registry in config/deploy.yml"
+        raise Kamal::ConfigurationError, "The primary_role #{primary_role_name} isn't defined"
       end
 
-      if raw_config.registry["password"].blank?
+      if primary_role.hosts.empty?
-        raise ArgumentError, "You must specify a password for the registry in config/deploy.yml (or set the ENV variable if that's used)"
+        raise Kamal::ConfigurationError, "No servers specified for the #{primary_role.name} primary_role"
       end
 
-      roles.each do |role|
+      unless allow_empty_roles?
-        if role.hosts.empty?
+        roles.each do |role|
-          raise ArgumentError, "No servers specified for the #{role.name} role"
+          if role.hosts.empty?
+            raise Kamal::ConfigurationError, "No servers specified for the #{role.name} role. You can ignore this with allow_empty_roles: true"
+          end
         end
       end
 
       true
     end
 
+    def ensure_valid_service_name
+      raise Kamal::ConfigurationError, "Service name can only include alphanumeric characters, hyphens, and underscores" unless raw_config[:service] =~ /^[a-z0-9_-]+$/i
+
+      true
+    end
+
     def ensure_valid_kamal_version
       if minimum_version && Gem::Version.new(minimum_version) > Gem::Version.new(Kamal::VERSION)
-        raise ArgumentError, "Current version is #{Kamal::VERSION}, minimum required is #{minimum_version}"
+        raise Kamal::ConfigurationError, "Current version is #{Kamal::VERSION}, minimum required is #{minimum_version}"
       end
 
       true
     end
 
+    def ensure_retain_containers_valid
+      raise Kamal::ConfigurationError, "Must retain at least 1 container" if retain_containers < 1
+
+      true
+    end
+
 
     def role_names
       raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort
@@ -272,7 +320,10 @@ class Kamal::Configuration
     def git_version
       @git_version ||=
         if Kamal::Git.used?
-          [ Kamal::Git.revision, Kamal::Git.uncommitted_changes.present? ? "_uncommitted_#{SecureRandom.hex(8)}" : "" ].join
+          if Kamal::Git.uncommitted_changes.present? && !builder.git_clone?
+            uncommitted_suffix = "_uncommitted_#{SecureRandom.hex(8)}"
+          end
+          [ Kamal::Git.revision, uncommitted_suffix ].compact.join
         else
           raise "Can't use commit hash as version, no git repository found in #{Dir.pwd}"
         end

lib/kamal/configuration/accessory.rb

@@ -1,30 +1,39 @@
 class Kamal::Configuration::Accessory
+  include Kamal::Configuration::Validation
+
   delegate :argumentize, :optionize, to: Kamal::Utils
 
-  attr_accessor :name, :specifics
+  attr_reader :name, :accessory_config, :env
 
   def initialize(name, config:)
-    @name, @config, @specifics = name.inquiry, config, config.raw_config["accessories"][name]
+    @name, @config, @accessory_config = name.inquiry, config, config.raw_config["accessories"][name]
+
+    validate! \
+      accessory_config,
+      example: validation_yml["accessories"]["mysql"],
+      context: "accessories/#{name}",
+      with: Kamal::Configuration::Validator::Accessory
+
+    @env = Kamal::Configuration::Env.new \
+      config: accessory_config.fetch("env", {}),
+      secrets: config.secrets,
+      context: "accessories/#{name}/env"
   end
 
   def service_name
-    "#{config.service}-#{name}"
+    accessory_config["service"] || "#{config.service}-#{name}"
   end
 
   def image
-    specifics["image"]
+    accessory_config["image"]
   end
 
   def hosts
-    if (specifics.keys & ["host", "hosts", "roles"]).size != 1
-      raise ArgumentError, "Specify one of `host`, `hosts` or `roles` for accessory `#{name}`"
-    end
-
     hosts_from_host || hosts_from_hosts || hosts_from_roles
   end
 
   def port
-    if port = specifics["port"]&.to_s
+    if port = accessory_config["port"]&.to_s
       port.include?(":") ? port : "#{port}:#{port}"
     end
   end
@@ -34,44 +43,40 @@ class Kamal::Configuration::Accessory
   end
 
   def labels
-    default_labels.merge(specifics["labels"] || {})
+    default_labels.merge(accessory_config["labels"] || {})
   end
 
   def label_args
     argumentize "--label", labels
   end
 
-  def env
-    specifics["env"] || {}
-  end
-
-  def env_file
-    Kamal::EnvFile.new(env)
-  end
-
-  def host_env_directory
-    File.join config.host_env_directory, "accessories"
-  end
-
-  def host_env_file_path
-    File.join host_env_directory, "#{service_name}.env"
-  end
-
   def env_args
-    argumentize "--env-file", host_env_file_path
+    [ *env.clear_args, *argumentize("--env-file", secrets_path) ]
+  end
+
+  def env_directory
+    File.join(config.env_directory, "accessories")
+  end
+
+  def secrets_io
+    env.secrets_io
+  end
+
+  def secrets_path
+    File.join(config.env_directory, "accessories", "#{service_name}.env")
   end
 
   def files
-    specifics["files"]&.to_h do |local_to_remote_mapping|
+    accessory_config["files"]&.to_h do |local_to_remote_mapping|
       local_file, remote_file = local_to_remote_mapping.split(":")
       [ expand_local_file(local_file), expand_remote_file(remote_file) ]
     end || {}
   end
 
   def directories
-    specifics["directories"]&.to_h do |host_to_container_mapping|
+    accessory_config["directories"]&.to_h do |host_to_container_mapping|
-      host_relative_path, container_path = host_to_container_mapping.split(":")
+      host_path, container_path = host_to_container_mapping.split(":")
-      [ expand_host_path(host_relative_path), container_path ]
+      [ expand_host_path(host_path), container_path ]
     end || {}
   end
 
@@ -84,7 +89,7 @@ class Kamal::Configuration::Accessory
   end
 
   def option_args
-    if args = specifics["options"]
+    if args = accessory_config["options"]
       optionize args
     else
       []
@@ -92,7 +97,7 @@ class Kamal::Configuration::Accessory
   end
 
   def cmd
-    specifics["cmd"]
+    accessory_config["cmd"]
   end
 
   private
@@ -111,10 +116,10 @@ class Kamal::Configuration::Accessory
     end
 
     def with_clear_env_loaded
-      (env["clear"] || env).each { |k, v| ENV[k] = v }
+      env.clear.each { |k, v| ENV[k] = v }
       yield
     ensure
-      (env["clear"] || env).each { |k, v| ENV.delete(k) }
+      env.clear.each { |k, v| ENV.delete(k) }
     end
 
     def read_dynamic_file(local_file)
@@ -126,25 +131,29 @@ class Kamal::Configuration::Accessory
     end
 
     def specific_volumes
-      specifics["volumes"] || []
+      accessory_config["volumes"] || []
     end
 
     def remote_files_as_volumes
-      specifics["files"]&.collect do |local_to_remote_mapping|
+      accessory_config["files"]&.collect do |local_to_remote_mapping|
         _, remote_file = local_to_remote_mapping.split(":")
         "#{service_data_directory + remote_file}:#{remote_file}"
       end || []
     end
 
     def remote_directories_as_volumes
-      specifics["directories"]&.collect do |host_to_container_mapping|
+      accessory_config["directories"]&.collect do |host_to_container_mapping|
-        host_relative_path, container_path = host_to_container_mapping.split(":")
+        host_path, container_path = host_to_container_mapping.split(":")
-        [ expand_host_path(host_relative_path), container_path ].join(":")
+        [ expand_host_path(host_path), container_path ].join(":")
       end || []
     end
 
-    def expand_host_path(host_relative_path)
+    def expand_host_path(host_path)
-      "#{service_data_directory}/#{host_relative_path}"
+      absolute_path?(host_path) ? host_path : File.join(service_data_directory, host_path)
+    end
+
+    def absolute_path?(path)
+      Pathname.new(path).absolute?
     end
 
     def service_data_directory
@@ -152,30 +161,16 @@ class Kamal::Configuration::Accessory
     end
 
     def hosts_from_host
-      if specifics.key?("host")
+      [ accessory_config["host"] ] if accessory_config.key?("host")
-        host = specifics["host"]
-        if host
-          [host]
-        else
-          raise ArgumentError, "Missing host for accessory `#{name}`"
-        end
-      end
     end
 
     def hosts_from_hosts
-      if specifics.key?("hosts")
+      accessory_config["hosts"] if accessory_config.key?("hosts")
-        hosts = specifics["hosts"]
-        if hosts.is_a?(Array)
-          hosts
-        else
-          raise ArgumentError, "Hosts should be an Array for accessory `#{name}`"
-        end
-      end
     end
 
     def hosts_from_roles
-      if specifics.key?("roles")
+      if accessory_config.key?("roles")
-        specifics["roles"].flat_map { |role| config.role(role).hosts }
+        accessory_config["roles"].flat_map { |role| config.role(role).hosts }
       end
     end
 end

lib/kamal/configuration/alias.rb

@@ -0,0 +1,15 @@
+class Kamal::Configuration::Alias
+  include Kamal::Configuration::Validation
+
+  attr_reader :name, :command
+
+  def initialize(name, config:)
+    @name, @command = name.inquiry, config.raw_config["aliases"][name]
+
+    validate! \
+      command,
+      example: validation_yml["aliases"]["uname"],
+      context: "aliases/#{name}",
+      with: Kamal::Configuration::Validator::Alias
+  end
+end

lib/kamal/configuration/boot.rb

@@ -1,20 +1,25 @@
 class Kamal::Configuration::Boot
+  include Kamal::Configuration::Validation
+
+  attr_reader :boot_config, :host_count
+
   def initialize(config:)
-    @options = config.raw_config.boot || {}
+    @boot_config = config.raw_config.boot || {}
     @host_count = config.all_hosts.count
+    validate! boot_config
   end
 
   def limit
-    limit = @options["limit"]
+    limit = boot_config["limit"]
 
     if limit.to_s.end_with?("%")
-      @host_count * limit.to_i / 100
+      [ host_count * limit.to_i / 100, 1 ].max
     else
       limit
     end
   end
 
   def wait
-    @options["wait"]
+    boot_config["wait"]
   end
 end

lib/kamal/configuration/builder.rb

@@ -1,67 +1,93 @@
 class Kamal::Configuration::Builder
-  def initialize(config:)
+  include Kamal::Configuration::Validation
-    @options = config.raw_config.builder || {}
-    @image = config.image
-    @server = config.registry["server"]
 
-    valid?
+  attr_reader :config, :builder_config
+  delegate :image, :service, to: :config
+  delegate :server, to: :"config.registry"
+
+  def initialize(config:)
+    @config = config
+    @builder_config = config.raw_config.builder || {}
+    @image = config.image
+    @server = config.registry.server
+    @service = config.service
+
+    validate! builder_config, with: Kamal::Configuration::Validator::Builder
   end
 
   def to_h
-    @options
+    builder_config
   end
 
-  def multiarch?
+  def remote
-    @options["multiarch"] != false
+    builder_config["remote"]
   end
 
-  def local?
+  def arches
-    !!@options["local"]
+    Array(builder_config.fetch("arch", default_arch))
+  end
+
+  def local_arches
+    @local_arches ||= if local_disabled?
+      []
+    elsif remote
+      arches & [ Kamal::Utils.docker_arch ]
+    else
+      arches
+    end
+  end
+
+  def remote_arches
+    @remote_arches ||= if remote
+      arches - local_arches
+    else
+      []
+    end
   end
 
   def remote?
-    !!@options["remote"]
+    remote_arches.any?
+  end
+
+  def local?
+    !local_disabled? && (arches.empty? || local_arches.any?)
   end
 
   def cached?
-    !!@options["cache"]
+    !!builder_config["cache"]
   end
 
   def args
-    @options["args"] || {}
+    builder_config["args"] || {}
   end
 
   def secrets
-    @options["secrets"] || []
+    (builder_config["secrets"] || []).to_h { |key| [ key, config.secrets[key] ] }
   end
 
   def dockerfile
-    @options["dockerfile"] || "Dockerfile"
+    builder_config["dockerfile"] || "Dockerfile"
+  end
+
+  def target
+    builder_config["target"]
   end
 
   def context
-    @options["context"] || "."
+    builder_config["context"] || "."
   end
 
-  def local_arch
+  def driver
-    @options["local"]["arch"] if local?
+    builder_config.fetch("driver", "docker-container")
   end
 
-  def local_host
+  def local_disabled?
-    @options["local"]["host"] if local?
+    builder_config["local"] == false
-  end
-
-  def remote_arch
-    @options["remote"]["arch"] if remote?
-  end
-
-  def remote_host
-    @options["remote"]["host"] if remote?
   end
 
   def cache_from
     if cached?
-      case @options["cache"]["type"]
+      case builder_config["cache"]["type"]
       when "gha"
         cache_from_config_for_gha
       when "registry"
@@ -72,7 +98,7 @@ class Kamal::Configuration::Builder
 
   def cache_to
     if cached?
-      case @options["cache"]["type"]
+      case builder_config["cache"]["type"]
       when "gha"
         cache_to_config_for_gha
       when "registry"
@@ -81,19 +107,50 @@ class Kamal::Configuration::Builder
     end
   end
 
+  def ssh
+    builder_config["ssh"]
+  end
+
+  def git_clone?
+    Kamal::Git.used? && builder_config["context"].nil?
+  end
+
+  def clone_directory
+    @clone_directory ||= File.join Dir.tmpdir, "kamal-clones", [ service, pwd_sha ].compact.join("-")
+  end
+
+  def build_directory
+    @build_directory ||=
+      if git_clone?
+        File.join clone_directory, repo_basename, repo_relative_pwd
+      else
+        "."
+      end
+  end
+
+  def docker_driver?
+    driver == "docker"
+  end
+
   private
     def valid?
+      if docker_driver?
+        raise ArgumentError, "Invalid builder configuration: the `docker` driver does not not support remote builders" if remote
+        raise ArgumentError, "Invalid builder configuration: the `docker` driver does not not support caching" if cached?
+        raise ArgumentError, "Invalid builder configuration: the `docker` driver does not not support multiple arches" if arches.many?
+      end
+
       if @options["cache"] && @options["cache"]["type"]
-        raise ArgumentError, "Invalid cache type: #{@options["cache"]["type"]}" unless ["gha", "registry"].include?(@options["cache"]["type"])
+        raise ArgumentError, "Invalid cache type: #{@options["cache"]["type"]}" unless [ "gha", "registry" ].include?(@options["cache"]["type"])
       end
     end
 
     def cache_image
-      @options["cache"]&.fetch("image", nil) || "#{@image}-build-cache"
+      builder_config["cache"]&.fetch("image", nil) || "#{image}-build-cache"
     end
 
     def cache_image_ref
-      [ @server, cache_image ].compact.join("/")
+      [ server, cache_image ].compact.join("/")
     end
 
     def cache_from_config_for_gha
@@ -105,10 +162,26 @@ class Kamal::Configuration::Builder
     end
 
     def cache_to_config_for_gha
-      [ "type=gha", @options["cache"]&.fetch("options", nil)].compact.join(",")
+      [ "type=gha", builder_config["cache"]&.fetch("options", nil) ].compact.join(",")
     end
 
     def cache_to_config_for_registry
-      [ "type=registry", @options["cache"]&.fetch("options", nil), "ref=#{cache_image_ref}" ].compact.join(",")
+      [ "type=registry", builder_config["cache"]&.fetch("options", nil), "ref=#{cache_image_ref}" ].compact.join(",")
+    end
+
+    def repo_basename
+      File.basename(Kamal::Git.root)
+    end
+
+    def repo_relative_pwd
+      Dir.pwd.delete_prefix(Kamal::Git.root)
+    end
+
+    def pwd_sha
+      Digest::SHA256.hexdigest(Dir.pwd)[0..12]
+    end
+
+    def default_arch
+      docker_driver? ? [] : [ "amd64", "arm64" ]
     end
 end

lib/kamal/configuration/docs/accessory.yml

@@ -0,0 +1,90 @@
+# Accessories
+#
+# Accessories can be booted on a single host, a list of hosts, or on specific roles.
+# The hosts do not need to be defined in the Kamal servers configuration.
+#
+# Accessories are managed separately from the main service - they are not updated
+# when you deploy and they do not have zero-downtime deployments.
+#
+# Run `kamal accessory boot <accessory>` to boot an accessory.
+# See `kamal accessory --help` for more information.
+
+# Configuring accessories
+#
+# First define the accessory in the `accessories`
+accessories:
+  mysql:
+
+    # Service name
+    #
+    # This is used in the service label and defaults to `<service>-<accessory>`
+    # where `<service>` is the main service name from the root configuration
+    service: mysql
+
+    # Image
+    #
+    # The Docker image to use, prefix with a registry if not using Docker hub
+    image: mysql:8.0
+
+    # Accessory hosts
+    #
+    # Specify one  of `host`, `hosts` or `roles`
+    host: mysql-db1
+    hosts:
+      - mysql-db1
+      - mysql-db2
+    roles:
+      - mysql
+
+    # Custom command
+    #
+    # You can set a custom command to run in the container, if you do not want to use the default
+    cmd: "bin/mysqld"
+
+    # Port mappings
+    #
+    # See https://docs.docker.com/network/, especially note the warning about the security
+    # implications of exposing ports publicly.
+    port: "127.0.0.1:3306:3306"
+
+    # Labels
+    labels:
+      app: myapp
+
+    # Options
+    # These are passed to the Docker run command in the form `--<name> <value>`
+    options:
+      restart: always
+      cpus: 2
+
+    # Environment variables
+    # See kamal docs env for more information
+    env:
+      ...
+
+    # Copying files
+    #
+    # You can specify files to mount into the container.
+    # The format is `local:remote` where `local` is the path to the file on the local machine
+    # and `remote` is the path to the file in the container.
+    #
+    # They will be uploaded from the local repo to the host and then mounted.
+    #
+    # ERB files will be evaluated before being copied.
+    files:
+      - config/my.cnf.erb:/etc/mysql/my.cnf
+      - config/myoptions.cnf:/etc/mysql/myoptions.cnf
+
+    # Directories
+    #
+    # You can specify directories to mount into the container. They will be created on the host
+    # before being mounted
+    directories:
+      - mysql-logs:/var/log/mysql
+
+    # Volumes
+    #
+    # Any other volumes to mount, in addition to the files and directories.
+    # They are not created or copied before mounting
+    volumes:
+      - /path/to/mysql-logs:/var/log/mysql

lib/kamal/configuration/docs/alias.yml

@@ -0,0 +1,26 @@
+# Aliases
+#
+# Aliases are shortcuts for Kamal commands.
+#
+# For example, for a Rails app, you might open a console with:
+#
+# ```shell
+# kamal app exec -i -r console "rails console"
+# ```
+#
+# By defining an alias, like this:
+aliases:
+  console: app exec -r console -i "rails console"
+# You can now open the console with:
+# ```shell
+# kamal console
+# ```
+
+# Configuring aliases
+#
+# Aliases are defined in the root config under the alias key
+#
+# Each alias is named and can only contain lowercase letters, numbers, dashes and underscores.
+
+aliases:
+  uname: app exec -p -q -r web "uname -a"

lib/kamal/configuration/docs/boot.yml

@@ -0,0 +1,19 @@
+# Booting
+#
+# When deploying to large numbers of hosts, you might prefer not to restart your services on every host at the same time.
+#
+# Kamal’s default is to boot new containers on all hosts in parallel. But you can control this with the boot configuration.
+
+# Fixed group sizes
+#
+# Here we boot 2 hosts at a time with a 10 second gap between each group.
+boot:
+  limit: 2
+  wait: 10
+
+# Percentage of hosts
+#
+# Here we boot 25% of the hosts at a time with a 2 second gap between each group.
+boot:
+  limit: 25%
+  wait: 2

lib/kamal/configuration/docs/builder.yml

@@ -0,0 +1,105 @@
+# Builder
+#
+# The builder configuration controls how the application is built with `docker build`
+#
+# If no configuration is specified, Kamal will:
+# 1. Create a buildx context called `kamal-local-docker-container`, using the docker-container driver
+# 2. Use `docker build` to build a multiarch image for linux/amd64,linux/arm64 with that context
+#
+# See https://kamal-deploy.org/docs/configuration/builder-examples/ for more information
+
+# Builder options
+#
+# Options go under the builder key in the root configuration.
+builder:
+  # Driver
+  #
+  # The build driver to use, defaults to `docker-container`
+  driver: docker
+
+  # Arch
+  #
+  # The architectures to build for, defaults to `[ amd64, arm64 ]`
+  # Unless you are using the docker driver, when it defaults to the local architecture
+  # You can set an array or just a single value
+  arch:
+    - amd64
+
+  # Remote configuration
+  #
+  # If you have a remote builder, you can configure it here
+  remote: ssh://docker@docker-builder
+
+  # Whether to allow local builds
+  #
+  # Defaults to true
+  local: true
+
+  # Builder cache
+  #
+  # The type must be either 'gha' or 'registry'
+  #
+  # The image is only used for registry cache. Not compatible with the docker driver
+  cache:
+    type: registry
+    options: mode=max
+    image: kamal-app-build-cache
+
+  # Build context
+  #
+  # If this is not set, then a local git clone of the repo is used.
+  # This ensures a clean build with no uncommitted changes.
+  #
+  # To use the local checkout instead you can set the context to `.`, or a path to another directory.
+  context: .
+
+  # Dockerfile
+  #
+  # The Dockerfile to use for building, defaults to `Dockerfile`
+  dockerfile: Dockerfile.production
+
+  # Build target
+  #
+  # If not set, then the default target is used
+  target: production
+
+  # Build Arguments
+  #
+  # Any additional build arguments, passed to `docker build` with `--build-arg <key>=<value>`
+  args:
+    ENVIRONMENT: production
+
+  # Referencing build arguments
+  #
+  # ```shell
+  # ARG RUBY_VERSION
+  # FROM ruby:$RUBY_VERSION-slim as base
+  # ```
+
+  # Build secrets
+  #
+  # Values are read from the environment.
+  #
+  secrets:
+    - SECRET1
+    - SECRET2
+
+  # Referencing Build Secrets
+  #
+  # ```shell
+  # # Copy Gemfiles
+  # COPY Gemfile Gemfile.lock ./
+  #
+  # # Install dependencies, including private repositories via access token
+  # # Then remove bundle cache with exposed GITHUB_TOKEN)
+  # RUN --mount=type=secret,id=GITHUB_TOKEN \
+  #   BUNDLE_GITHUB__COM=x-access-token:$(cat /run/secrets/GITHUB_TOKEN) \
+  #   bundle install && \
+  #   rm -rf /usr/local/bundle/cache
+  # ```
+
+
+  # SSH
+  #
+  # SSH agent socket or keys to expose to the build
+  ssh: default=$SSH_AUTH_SOCK

lib/kamal/configuration/docs/configuration.yml

@@ -0,0 +1,180 @@
+# Kamal Configuration
+#
+# Configuration is read from the `config/deploy.yml`
+#
+
+# Destinations
+#
+# When running commands, you can specify a destination with the `-d` flag,
+# e.g. `kamal deploy -d staging`
+#
+# In this case the configuration will also be read from `config/deploy.staging.yml`
+# and merged with the base configuration.
+
+# Extensions
+#
+# Kamal will not accept unrecognized keys in the configuration file.
+#
+# However, you might want to declare a configuration block using YAML anchors
+# and aliases to avoid repetition.
+#
+# You can use prefix a configuration section with `x-` to indicate that it is an
+# extension. Kamal will ignore the extension and not raise an error.
+
+# The service name
+# This is a required value. It is used as the container name prefix.
+service: myapp
+
+# The Docker image name
+#
+# The image will be pushed to the configured registry.
+image: my-image
+
+# Labels
+#
+# Additional labels to add to the container
+labels:
+  my-label: my-value
+
+# Additional volumes to mount into the container
+volumes:
+  - /path/on/host:/path/in/container:ro
+
+# Registry
+#
+# The Docker registry configuration, see kamal docs registry
+registry:
+  ...
+
+# Servers
+#
+# The servers to deploy to, optionally with custom roles, see kamal docs servers
+servers:
+  ...
+
+# Environment variables
+#
+# See kamal docs env
+env:
+  ...
+
+# Asset Bridging
+#
+# Used for asset bridging across deployments, default to `nil`
+#
+# If there are changes to CSS or JS files, we may get requests
+# for the old versions on the new container and vice-versa.
+#
+# To avoid 404s we can specify an asset path.
+# Kamal will replace that path in the container with a mapped
+# volume containing both sets of files.
+# This requires that file names change when the contents change
+# (e.g. by including a hash of the contents in the name).
+
+# To configure this, set the path to the assets:
+asset_path: /path/to/assets
+
+# Path to hooks, defaults to `.kamal/hooks`
+# See https://kamal-deploy.org/docs/hooks for more information
+hooks_path: /user_home/kamal/hooks
+
+# Require destinations
+#
+# Whether deployments require a destination to be specified, defaults to `false`
+require_destination: true
+
+# The primary role
+#
+# This defaults to `web`, but if you have no web role, you can change this
+primary_role: workers
+
+# Allowing empty roles
+#
+# Whether roles with no servers are allowed. Defaults to `false`.
+allow_empty_roles: false
+
+# Stop wait time
+#
+# How long we wait for a container to stop before killing it, defaults to 30 seconds
+stop_wait_time: 60
+
+# Retain containers
+#
+# How many old containers and images we retain, defaults to 5
+retain_containers: 3
+
+# Minimum version
+#
+# The minimum version of Kamal required to deploy this configuration, defaults to nil
+minimum_version: 1.3.0
+
+# Readiness delay
+#
+# Seconds to wait for a container to boot after is running, default 7
+# This only applies to containers that do not specify a healthcheck
+readiness_delay: 4
+
+# Run directory
+#
+# Directory to store kamal runtime files in on the host, default `.kamal`
+run_directory: /etc/kamal
+
+# SSH options
+#
+# See kamal docs ssh
+ssh:
+  ...
+
+# Builder options
+#
+# See kamal docs builder
+builder:
+  ...
+
+# Accessories
+#
+# Additionals services to run in Docker, see kamal docs accessory
+accessories:
+  ...
+
+# Traefik
+#
+# The Traefik proxy is used for zero-downtime deployments, see kamal docs traefik
+traefik:
+  ...
+
+# Proxy
+#
+# **Experimental** Configuration for kamal-proxy the replacement for Traefik, see kamal docs proxy
+proxy:
+  ...
+
+# SSHKit
+#
+# See kamal docs sshkit
+sshkit:
+  ...
+
+# Boot options
+#
+# See kamal docs boot
+boot:
+  ...
+
+# Healthcheck
+#
+# Configuring healthcheck commands, intervals and timeouts, see kamal docs healthcheck
+healthcheck:
+  ...
+
+# Logging
+#
+# Docker logging configuration, see kamal docs logging
+logging:
+  ...
+
+# Aliases
+#
+# Alias configuration, see kamal docs alias
+aliases:
+  ...

lib/kamal/configuration/docs/env.yml

@@ -0,0 +1,70 @@
+# Environment variables
+#
+# Environment variables can be set directly in the Kamal configuration or
+# loaded from a .env file, for secrets that should not be checked into Git.
+
+# Reading environment variables from the configuration
+#
+# Environment variables can be set directly in the configuration file.
+#
+# These are passed to the docker run command when deploying.
+env:
+  DATABASE_HOST: mysql-db1
+  DATABASE_PORT: 3306
+
+# Using .env file to load required environment variables
+#
+# Kamal uses dotenv to automatically load environment variables set in the .env file present
+# in the application root.
+#
+# This file can be used to set variables like KAMAL_REGISTRY_PASSWORD or database passwords.
+# But for this reason you must ensure that .env files are not checked into Git or included
+# in your Dockerfile! The format is just key-value like:
+# ```
+# KAMAL_REGISTRY_PASSWORD=pw
+# DB_PASSWORD=secret123
+# ```
+#
+# To pass the secrets you should list them under the `secret` key. When you do this the
+# other variables need to be moved under the `clear` key.
+#
+# Unlike clear values, secrets are not passed directly to the container,
+# but are stored in an env file on the host
+env:
+  clear:
+    DB_USER: app
+  secret:
+    - DB_PASSWORD
+
+# Tags
+#
+# Tags are used to add extra env variables to specific hosts.
+# See kamal docs servers for how to tag hosts.
+#
+# Tags are only allowed in the top level env configuration (i.e not under a role specific env).
+#
+# The env variables can be specified with secret and clear values as explained above.
+env:
+  tags:
+    <tag1>:
+      MYSQL_USER: monitoring
+    <tag2>:
+      clear:
+        MYSQL_USER: readonly
+      secret:
+        - MYSQL_PASSWORD
+
+# Example configuration
+env:
+  clear:
+    MYSQL_USER: app
+  secret:
+    - MYSQL_PASSWORD
+  tags:
+    monitoring:
+      MYSQL_USER: monitoring
+    replica:
+      clear:
+        MYSQL_USER: readonly
+      secret:
+        - READONLY_PASSWORD

lib/kamal/configuration/docs/healthcheck.yml

@@ -0,0 +1,59 @@
+# Healthcheck configuration
+#
+# On roles that are running Traefik, Kamal will supply a default healthcheck to `docker run`.
+# For other roles, by default no healthcheck is supplied.
+#
+# If no healthcheck is supplied and the image does not define one, then we wait for the container
+# to reach a running state and then pause for the readiness delay.
+#
+# The default healthcheck is `curl -f http://localhost:<port>/<path>`, so it assumes that `curl`
+# is available within the container.
+
+# Healthcheck options
+#
+# These go under the `healthcheck` key in the root or role configuration.
+healthcheck:
+
+  # Command
+  #
+  # The command to run, defaults to `curl -f http://localhost:<port>/<path>` on roles running Traefik
+  cmd: "curl -f http://localhost"
+
+  # Interval
+  #
+  # The Docker healthcheck interval, defaults to `1s`
+  interval: 10s
+
+  # Max attempts
+  #
+  # The maximum number of times we poll the container to see if it is healthy, defaults to `7`
+  # Each check is separated by an increasing interval starting with 1 second.
+  max_attempts: 3
+
+  # Port
+  #
+  # The port to use in the healthcheck, defaults to `3000`
+  port: "80"
+
+  # Path
+  #
+  # The path to use in the healthcheck, defaults to `/up`
+  path: /health
+
+  # Cords for zero-downtime deployments
+  #
+  # The cord file is used for zero-downtime deployments. The healthcheck is augmented with a check
+  # for the existance of the file. This allows us to delete the file and force the container to
+  # become unhealthy, causing Traefik to stop routing traffic to it.
+  #
+  # Kamal mounts a volume at this location and creates the file before starting the container.
+  # You can set the value to `false` to disable the cord file, but this loses the zero-downtime
+  # guarantee.
+  #
+  # The default value is `/tmp/kamal-cord`
+  cord: /cord
+
+  # Log lines
+  #
+  # Number of lines to log from the container when the healthcheck fails, defaults to `50`
+  log_lines: 100

lib/kamal/configuration/docs/logging.yml

@@ -0,0 +1,21 @@
+# Custom logging configuration
+#
+# Set these to control the Docker logging driver and options.
+
+# Logging settings
+#
+# These go under the logging key in the configuration file.
+#
+# This can be specified in the root level or for a specific role.
+logging:
+
+  # Driver
+  #
+  # The logging driver to use, passed to Docker via `--log-driver`
+  driver: json-file
+
+  # Options
+  #
+  # Any logging options to pass to the driver, passed to Docker via `--log-opt`
+  options:
+    max-size: 100m

lib/kamal/configuration/docs/proxy.yml

@@ -0,0 +1,127 @@
+# Proxy
+#
+# **Experimental** [kamal-proxy](http://github.com/basecamp/kamal-proxy) is a
+# custom built specifically for Kamal. It will replace Traefik in Kamal v2.0,
+# but currently is available as an experimental feature.
+#
+# When this is enabled, the proxy will be started on the hosts listed under the hosts key.
+# In addition, the kamal traefik command will be disabled and replaced by kamal proxy.
+#
+# The kamal proxy command works identically to kamal traefik on hosts that have not
+# been included. It will also handle switching between Traefik and kamal-proxy when you
+# run kamal proxy reboot.
+
+# Limitations
+#
+# Currently the proxy will run on ports 80 and 443 and will bind to those
+# ports on the host.
+#
+# There is no way to set custom options for `docker run` when booting the proxy.
+#
+# If you have custom Traefik configuration via labels or boot arguments they may
+# not have an equivalent in kamal-proxy.
+
+# Proxy settings
+#
+# The proxy is configured in the root configuration under `traefik`. These are
+# options that are set when deploying the application, not when booting the proxy
+#
+# They are application specific, so are not shared when multiple applications
+# with the same proxy.
+proxy:
+
+  # Enabled
+  #
+  # Whether to enable experimental proxy support. Defaults to false
+  enabled: true
+
+  # Hosts
+  #
+  # The hosts to run the proxy on, instead of Traefik
+  # This is a temporary setting and will be removed when we full switch to kamal-proxy
+  #
+  # If you run `kamal traefik reboot`, then the proxy will be started on these hosts
+  # in place of traefik.
+  hosts:
+    - 10.0.0.1
+    - 10.0.0.2
+
+  # Host
+  #
+  # This is the host that will be used to serve the app. By setting this you can run
+  # multiple apps on the same server sharing the same instance of the proxy.
+  #
+  # If this is set only requests that match this host will be forwarded by the proxy.
+  # if this is not set, then all requests will be forwarded, except for matching
+  # requests for other apps that do have a host set.
+  host: foo.example.com
+
+  # App port
+  #
+  # The port the application container is exposed on
+  # Defaults to 80
+  app_port: 3000
+
+  # SSL
+  #
+  # Kamal Proxy can automatically obtain and renew TLS certificates for your applications.
+  # To ensure this set, the ssl flag. This only works if we are deploying to one server and
+  # the host flag is set.
+  ssl: true
+
+  # Deploy timeout
+  #
+  # How long to wait for the app to boot when deploying, defaults to 30 seconds
+  deploy_timeout: 10s
+
+  # Response timeout
+  #
+  # How long to wait for requests to complete before timing out, defaults to 30 seconds
+  response_timeout: 10
+
+  # Healthcheck
+  #
+  # When deploying, the proxy will by default hit /up once every second until we hit
+  # the deploy timeout, with a 5 second timeout for each request.
+  #
+  # Once the app is up, the proxy will stop hitting the healthcheck endpoint.
+  healthcheck:
+    interval: 3
+    path: /health
+    timeout: 3
+
+  # Buffering
+  #
+  # Whether to buffer request and response bodies in the proxy
+  #
+  # By default buffering is enabled with a max request body size of 1GB and no limit
+  # for response size.
+  #
+  # You can also set the memory limit for buffering, which defaults to 1MB, anything
+  # larger than that is written to disk.
+  buffering:
+    requests: true
+    responses: true
+    max_request_body: 40_000_000
+    max_response_body: 0
+    memory: 2_000_000
+
+  # Logging
+  #
+  # Configure request logging for the proxy
+  # You can specify request and response headers to log.
+  # By default, Cache-Control and Last-Modified request headers are logged
+  logging:
+    request_headers:
+      - Cache-Control
+      - X-Forwarded-Proto
+    response_headers:
+      - X-Request-ID
+      - X-Request-Start
+
+  # Forward headers
+  #
+  # Whether to forward the X-Forwarded-For and X-Forwarded-Proto headers (defaults to false)
+  #
+  # If you are behind a trusted proxy, you can set this to true to forward the headers.
+  forward_headers: true

lib/kamal/configuration/docs/registry.yml

@@ -0,0 +1,49 @@
+# Registry
+#
+# The default registry is Docker Hub, but you can change it using registry/server:
+#
+# A reference to secret (in this case DOCKER_REGISTRY_TOKEN) will look up the secret
+# in the local environment.
+
+registry:
+  server: registry.digitalocean.com
+  username:
+    - DOCKER_REGISTRY_TOKEN
+  password:
+    - DOCKER_REGISTRY_TOKEN
+
+# Using AWS ECR as the container registry
+# You will need to have the aws CLI installed locally for this to work.
+# AWS ECR’s access token is only valid for 12hrs. In order to not have to manually regenerate the token every time, you can use ERB in the deploy.yml file to shell out to the aws cli command, and obtain the token:
+
+registry:
+  server: <your aws account id>.dkr.ecr.<your aws region id>.amazonaws.com
+  username: AWS
+  password: <%= %x(aws ecr get-login-password) %>
+
+# Using GCP Artifact Registry as the container registry
+# To sign into Artifact Registry, you would need to
+# [create a service account](https://cloud.google.com/iam/docs/service-accounts-create#creating)
+# and [set up roles and permissions](https://cloud.google.com/artifact-registry/docs/access-control#permissions).
+# Normally, assigning a roles/artifactregistry.writer role should be sufficient.
+#
+# Once the service account is ready, you need to generate and download a JSON key, base64 encode it and add to .env:
+#
+# ```shell
+# echo "KAMAL_REGISTRY_PASSWORD=$(base64 -i /path/to/key.json)" | tr -d "\\n"  >> .env
+# ```
+# Use the env variable as password along with _json_key_base64 as username.
+# Here’s the final configuration:
+
+registry:
+  server: <your registry region>-docker.pkg.dev
+  username: _json_key_base64
+  password:
+    - KAMAL_REGISTRY_PASSWORD
+
+# Validating the configuration
+#
+# You can validate the configuration by running:
+# ```shell
+# kamal registry login
+# ```

lib/kamal/configuration/docs/role.yml

@@ -0,0 +1,52 @@
+# Roles
+#
+# Roles are used to configure different types of servers in the deployment.
+# The most common use for this is to run a web servers and job servers.
+#
+# Kamal expects there to be a `web` role, unless you set a different `primary_role`
+# in the root configuration.
+
+# Role configuration
+#
+# Roles are specified under the servers key
+servers:
+
+  # Simple role configuration
+  #
+  #
+  # This can be a list of hosts, if you don't need custom configuration for the role.
+  #
+  # You can set tags on the hosts for custom env variables (see kamal docs env)
+  web:
+    - 172.1.0.1
+    - 172.1.0.2: experiment1
+    - 172.1.0.2: [ experiment1, experiment2 ]
+
+  # Custom role configuration
+  #
+  # When there are other options to set, the list of hosts goes under the `hosts` key
+  #
+  # By default only the primary role uses Traefik, but you can set `traefik` to change
+  # it.
+  #
+  # You can also set a custom cmd to run in the container, and overwrite other settings
+  # from the root configuration.
+  workers:
+    hosts:
+      - 172.1.0.3
+      - 172.1.0.4: experiment1
+    traefik: true
+    cmd: "bin/jobs"
+    options:
+      memory: 2g
+      cpus: 4
+    healthcheck:
+      ...
+    logging:
+      ...
+    labels:
+      my-label: workers
+    env:
+      ...
+    asset_path: /public
+

lib/kamal/configuration/docs/servers.yml

@@ -0,0 +1,27 @@
+# Servers
+#
+# Servers are split into different roles, with each role having its own configuration.
+#
+# For simpler deployments though where all servers are identical, you can just specify a list of servers
+# They will be implicitly assigned to the `web` role.
+servers:
+  - 172.0.0.1
+  - 172.0.0.2
+  - 172.0.0.3
+
+# Tagging servers
+#
+# Servers can be tagged, with the tags used to add custom env variables (see kamal docs env).
+servers:
+  - 172.0.0.1
+  - 172.0.0.2: experiments
+  - 172.0.0.3: [ experiments, three ]
+
+# Roles
+#
+# For more complex deployments (e.g. if you are running job hosts), you can specify roles, and configure each separately (see kamal docs role)
+servers:
+  web:
+    ...
+  workers:
+    ...

lib/kamal/configuration/docs/ssh.yml

@@ -0,0 +1,66 @@
+# SSH configuration
+#
+# Kamal uses SSH to connect run commands on your hosts.
+# By default it will attempt to connect to the root user on port 22
+#
+# If you are using non-root user, you may need to bootstrap your servers manually, before using them with Kamal. On Ubuntu, you’d do:
+#
+# ```shell
+# sudo apt update
+# sudo apt upgrade -y
+# sudo apt install -y docker.io curl git
+# sudo usermod -a -G docker app
+# ```
+
+
+# SSH options
+#
+# The options are specified under the ssh key in the configuration file.
+ssh:
+
+  # The SSH user
+  #
+  # Defaults to `root`
+  #
+  user: app
+
+  # The SSH port
+  #
+  # Defaults to 22
+  port: "2222"
+
+  # Proxy host
+  #
+  # Specified in the form <host> or <user>@<host>
+  proxy: root@proxy-host
+
+  # Proxy command
+  #
+  # A custom proxy command, required for older versions of SSH
+  proxy_command: "ssh -W %h:%p user@proxy"
+
+  # Log level
+  #
+  # Defaults to `fatal`. Set this to debug if you are having
+  # SSH connection issues.
+  log_level: debug
+
+  # Keys Only
+  #
+  # Set to true to use only private keys from keys and key_data parameters, 
+  # even if ssh-agent offers more identities. This option is intended for 
+  # situations where ssh-agent offers many different identites or you have 
+  # a need to overwrite all identites and force a single one.
+  keys_only: false
+
+  # Keys
+  #
+  # An array of file names of private keys to use for publickey
+  # and hostbased authentication
+  keys: [ "~/.ssh/id.pem" ]
+
+  # Key Data
+  #
+  # An array of strings, with each element of the array being
+  # a raw private key in PEM format.
+  key_data: [ "-----BEGIN OPENSSH PRIVATE KEY-----" ]

lib/kamal/configuration/docs/sshkit.yml

@@ -0,0 +1,23 @@
+# SSHKit
+#
+# [SSHKit](https://github.com/capistrano/sshkit) is the SSH toolkit used by Kamal.
+#
+# The default settings should be sufficient for most use cases, but
+# when connecting to a large number of hosts you may need to adjust
+
+# SSHKit options
+#
+# The options are specified under the sshkit key in the configuration file.
+sshkit:
+
+  # Max concurrent starts
+  #
+  # Creating SSH connections concurrently can be an issue when deploying to many servers.
+  # By default Kamal will limit concurrent connection starts to 30 at a time.
+  max_concurrent_starts: 10
+
+  # Pool idle timeout
+  #
+  # Kamal sets a long idle timeout of 900 seconds on connections to try to avoid
+  # re-connection storms after an idle period, like building an image or waiting for CI.
+  pool_idle_timeout: 300

lib/kamal/configuration/docs/traefik.yml

@@ -0,0 +1,62 @@
+# Traefik
+#
+# Traefik is a reverse proxy, used by Kamal for zero-downtime deployments.
+#
+# We start an instance on the hosts in it's own container.
+#
+# During a deployment:
+# 1. We start a new container which Traefik automatically detects due to the labels we have applied
+# 2. Traefik starts routing traffic to the new container
+# 3. We force the old container to fail it's healthcheck, causing Traefik to stop routing traffic to it
+# 4. We stop the old container
+
+# Traefik settings
+#
+# Traekik is configured in the root configuration under `traefik`.
+traefik:
+
+  # Image
+  #
+  # The Traefik image to use, defaults to `traefik:v2.10`
+  image: traefik:v2.9
+
+  # Host port
+  #
+  # The host port to publish the Traefik container on, defaults to `80`
+  host_port: "8080"
+
+  # Disabling publishing
+  #
+  # To avoid publishing the Traefik container, set this to `false`
+  publish: false
+
+  # Labels
+  #
+  # Additional labels to apply to the Traefik container
+  labels:
+    traefik.http.routers.catchall.entryPoints: http
+    traefik.http.routers.catchall.rule: PathPrefix(`/`)
+    traefik.http.routers.catchall.service: unavailable
+    traefik.http.routers.catchall.priority: "1"
+    traefik.http.services.unavailable.loadbalancer.server.port: "0"
+
+  # Arguments
+  #
+  # Additional arguments to pass to the Traefik container
+  args:
+    entryPoints.http.address: ":80"
+    entryPoints.http.forwardedHeaders.insecure: true
+    accesslog: true
+    accesslog.format: json
+
+  # Options
+  #
+  # Additional options to pass to `docker run`
+  options:
+    cpus: 2
+
+  # Environment variables
+  #
+  # See kamal docs env
+  env:
+    ...

lib/kamal/configuration/env.rb

@@ -0,0 +1,29 @@
+class Kamal::Configuration::Env
+  include Kamal::Configuration::Validation
+
+  attr_reader :context, :secrets
+  attr_reader :clear, :secret_keys
+  delegate :argumentize, to: Kamal::Utils
+
+  def initialize(config:, secrets:, context: "env")
+    @clear = config.fetch("clear", config.key?("secret") || config.key?("tags") ? {} : config)
+    @secrets = secrets
+    @secret_keys = config.fetch("secret", [])
+    @context = context
+    validate! config, context: context, with: Kamal::Configuration::Validator::Env
+  end
+
+  def clear_args
+    argumentize("--env", clear)
+  end
+
+  def secrets_io
+    Kamal::EnvFile.new(secret_keys.to_h { |key| [ key, secrets[key] ] }).to_io
+  end
+
+  def merge(other)
+    self.class.new \
+      config: { "clear" => clear.merge(other.clear), "secret" => secret_keys | other.secret_keys },
+      secrets: secrets
+  end
+end

lib/kamal/configuration/env/tag.rb

@@ -0,0 +1,13 @@
+class Kamal::Configuration::Env::Tag
+  attr_reader :name, :config, :secrets
+
+  def initialize(name, config:, secrets:)
+    @name = name
+    @config = config
+    @secrets = secrets
+  end
+
+  def env
+    Kamal::Configuration::Env.new(config: config, secrets: secrets)
+  end
+end

lib/kamal/configuration/healthcheck.rb

@@ -0,0 +1,63 @@
+class Kamal::Configuration::Healthcheck
+  include Kamal::Configuration::Validation
+
+  attr_reader :healthcheck_config
+
+  def initialize(healthcheck_config:, context: "healthcheck")
+    @healthcheck_config = healthcheck_config || {}
+    validate! @healthcheck_config, context: context
+  end
+
+  def merge(other)
+    self.class.new healthcheck_config: healthcheck_config.deep_merge(other.healthcheck_config)
+  end
+
+  def cmd
+    healthcheck_config.fetch("cmd", http_health_check)
+  end
+
+  def port
+    healthcheck_config.fetch("port", 3000)
+  end
+
+  def path
+    healthcheck_config.fetch("path", "/up")
+  end
+
+  def max_attempts
+    healthcheck_config.fetch("max_attempts", 7)
+  end
+
+  def interval
+    healthcheck_config.fetch("interval", "1s")
+  end
+
+  def cord
+    healthcheck_config.fetch("cord", "/tmp/kamal-cord")
+  end
+
+  def log_lines
+    healthcheck_config.fetch("log_lines", 50)
+  end
+
+  def set_port_or_path?
+    healthcheck_config["port"].present? || healthcheck_config["path"].present?
+  end
+
+  def to_h
+    {
+      "cmd" => cmd,
+      "interval" => interval,
+      "max_attempts" => max_attempts,
+      "port" => port,
+      "path" => path,
+      "cord" => cord,
+      "log_lines" => log_lines
+    }
+  end
+
+  private
+    def http_health_check
+      "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present?
+    end
+end

lib/kamal/configuration/logging.rb

@@ -0,0 +1,33 @@
+class Kamal::Configuration::Logging
+  delegate :optionize, :argumentize, to: Kamal::Utils
+
+  include Kamal::Configuration::Validation
+
+  attr_reader :logging_config
+
+  def initialize(logging_config:, context: "logging")
+    @logging_config = logging_config || {}
+    validate! @logging_config, context: context
+  end
+
+  def driver
+    logging_config["driver"]
+  end
+
+  def options
+    logging_config.fetch("options", {})
+  end
+
+  def merge(other)
+    self.class.new logging_config: logging_config.deep_merge(other.logging_config)
+  end
+
+  def args
+    if driver.present? || options.present?
+      optionize({ "log-driver" => driver }.compact) +
+        argumentize("--log-opt", options)
+    else
+      argumentize("--log-opt", { "max-size" => "10m" })
+    end
+  end
+end

lib/kamal/configuration/proxy.rb

@@ -0,0 +1,80 @@
+class Kamal::Configuration::Proxy
+  include Kamal::Configuration::Validation
+
+  DEFAULT_HTTP_PORT = 80
+  DEFAULT_HTTPS_PORT = 443
+  DEFAULT_IMAGE = "basecamp/kamal-proxy:latest"
+  DEFAULT_LOG_REQUEST_HEADERS = [ "Cache-Control", "Last-Modified" ]
+
+  delegate :argumentize, :optionize, to: Kamal::Utils
+
+  def initialize(config:)
+    @config = config
+    @proxy_config = config.raw_config.proxy || {}
+    validate! proxy_config, with: Kamal::Configuration::Validator::Proxy
+  end
+
+  def enabled?
+    !!proxy_config.fetch("enabled", false)
+  end
+
+  def hosts
+    if enabled?
+      proxy_config.fetch("hosts", [])
+    else
+      []
+    end
+  end
+
+  def app_port
+    proxy_config.fetch("app_port", 80)
+  end
+
+  def image
+    proxy_config.fetch("image", DEFAULT_IMAGE)
+  end
+
+  def container_name
+    "kamal-proxy"
+  end
+
+  def publish_args
+    argumentize "--publish", [ "#{DEFAULT_HTTP_PORT}:#{DEFAULT_HTTP_PORT}", "#{DEFAULT_HTTPS_PORT}:#{DEFAULT_HTTPS_PORT}" ]
+  end
+
+  def ssl?
+    proxy_config.fetch("ssl", false)
+  end
+
+  def deploy_options
+    {
+      host: proxy_config["host"],
+      tls: proxy_config["ssl"],
+      "deploy-timeout": proxy_config["deploy_timeout"],
+      "drain-timeout": proxy_config["drain_timeout"],
+      "health-check-interval": proxy_config.dig("health_check", "interval"),
+      "health-check-timeout": proxy_config.dig("health_check", "timeout"),
+      "health-check-path": proxy_config.dig("health_check", "path"),
+      "target-timeout": proxy_config["response_timeout"],
+      "buffer-requests": proxy_config.fetch("buffering", { "requests": true }).fetch("requests", true),
+      "buffer-responses": proxy_config.fetch("buffering", { "responses": true }).fetch("responses", true),
+      "buffer-memory": proxy_config.dig("buffering", "memory"),
+      "max-request-body": proxy_config.dig("buffering", "max_request_body"),
+      "max-response-body": proxy_config.dig("buffering", "max_response_body"),
+      "forward-headers": proxy_config.dig("forward_headers"),
+      "log-request-header": proxy_config.dig("logging", "request_headers") || DEFAULT_LOG_REQUEST_HEADERS,
+      "log-response-header": proxy_config.dig("logging", "response_headers")
+    }.compact
+  end
+
+  def deploy_command_args
+    optionize deploy_options
+  end
+
+  def config_directory_as_docker_volume
+    File.join config.run_directory_as_docker_volume, "proxy", "config"
+  end
+
+  private
+    attr_reader :config, :proxy_config
+end

lib/kamal/configuration/registry.rb

@@ -0,0 +1,32 @@
+class Kamal::Configuration::Registry
+  include Kamal::Configuration::Validation
+
+  attr_reader :registry_config, :secrets
+
+  def initialize(config:)
+    @registry_config = config.raw_config.registry || {}
+    @secrets = config.secrets
+    validate! registry_config, with: Kamal::Configuration::Validator::Registry
+  end
+
+  def server
+    registry_config["server"]
+  end
+
+  def username
+    lookup("username")
+  end
+
+  def password
+    lookup("password")
+  end
+
+  private
+    def lookup(key)
+      if registry_config[key].is_a?(Array)
+        secrets[registry_config[key].first]
+      else
+        registry_config[key]
+      end
+    end
+end

lib/kamal/configuration/role.rb

@@ -1,11 +1,33 @@
 class Kamal::Configuration::Role
+  include Kamal::Configuration::Validation
+
   CORD_FILE = "cord"
   delegate :argumentize, :optionize, to: Kamal::Utils
 
-  attr_accessor :name
+  attr_reader :name, :config, :specialized_env, :specialized_logging, :specialized_healthcheck
+
+  alias to_s name
 
   def initialize(name, config:)
-   @name, @config = name.inquiry, config
+    @name, @config = name.inquiry, config
+    validate! \
+      specializations,
+      example: validation_yml["servers"]["workers"],
+      context: "servers/#{name}",
+      with: Kamal::Configuration::Validator::Role
+
+    @specialized_env = Kamal::Configuration::Env.new \
+      config: specializations.fetch("env", {}),
+      secrets: config.secrets,
+      context: "servers/#{name}/env"
+
+    @specialized_logging = Kamal::Configuration::Logging.new \
+      logging_config: specializations.fetch("logging", {}),
+      context: "servers/#{name}/logging"
+
+    @specialized_healthcheck = Kamal::Configuration::Healthcheck.new \
+      healthcheck_config: specializations.fetch("healthcheck", {}),
+      context: "servers/#{name}/healthcheck"
   end
 
   def primary_host
@@ -13,7 +35,11 @@ class Kamal::Configuration::Role
   end
 
   def hosts
-    @hosts ||= extract_hosts_from_config
+    tagged_hosts.keys
+  end
+
+  def env_tags(host)
+    tagged_hosts.fetch(host).collect { |tag| config.env_tag(tag) }
   end
 
   def cmd
@@ -32,33 +58,46 @@ class Kamal::Configuration::Role
     default_labels.merge(traefik_labels).merge(custom_labels)
   end
 
+  def labels_for_proxy
+    default_labels.merge(custom_labels)
+  end
+
   def label_args
     argumentize "--label", labels
   end
 
-
+  def label_args_for_proxy
-  def env
+    argumentize "--label", labels_for_proxy
-    if config.env && config.env["secret"]
-      merged_env_with_secrets
-    else
-      merged_env
-    end
   end
 
-  def env_file
+  def logging_args
-    Kamal::EnvFile.new(env)
+    logging.args
   end
 
-  def host_env_directory
+  def logging
-    File.join config.host_env_directory, "roles"
+    @logging ||= config.logging.merge(specialized_logging)
   end
 
-  def host_env_file_path
+
-    File.join host_env_directory, "#{[config.service, name, config.destination].compact.join("-")}.env"
+  def env(host)
+    @envs ||= {}
+    @envs[host] ||= [ config.env, specialized_env, *env_tags(host).map(&:env) ].reduce(:merge)
   end
 
-  def env_args
+  def env_args(host)
-    argumentize "--env-file", host_env_file_path
+    [ *env(host).clear_args, *argumentize("--env-file", secrets_path) ]
+  end
+
+  def env_directory
+    File.join(config.env_directory, "roles")
+  end
+
+  def secrets_io(host)
+    env(host).secrets_io
+  end
+
+  def secrets_path
+    File.join(config.env_directory, "roles", "#{container_prefix}.env")
   end
 
   def asset_volume_args
@@ -67,48 +106,57 @@ class Kamal::Configuration::Role
 
 
   def health_check_args(cord: true)
-    if health_check_cmd.present?
+    if running_traefik? || healthcheck.set_port_or_path?
       if cord && uses_cord?
-        optionize({ "health-cmd" => health_check_cmd_with_cord, "health-interval" => health_check_interval })
+        optionize({ "health-cmd" => health_check_cmd_with_cord, "health-interval" => healthcheck.interval })
           .concat(cord_volume.docker_args)
       else
-        optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval })
+        optionize({ "health-cmd" => healthcheck.cmd, "health-interval" => healthcheck.interval })
       end
     else
       []
     end
   end
 
-  def health_check_cmd
+  def healthcheck
-    health_check_options["cmd"] || http_health_check(port: health_check_options["port"], path: health_check_options["path"])
+    @healthcheck ||=
+      if running_traefik?
+        config.healthcheck.merge(specialized_healthcheck)
+      else
+        specialized_healthcheck
+      end
   end
 
   def health_check_cmd_with_cord
-    "(#{health_check_cmd}) && (stat #{cord_container_file} > /dev/null || exit 1)"
+    "(#{healthcheck.cmd}) && (stat #{cord_container_file} > /dev/null || exit 1)"
-  end
-
-  def health_check_interval
-    health_check_options["interval"] || "1s"
   end
 
 
   def running_traefik?
-    name.web? || specializations["traefik"]
+    if specializations["traefik"].nil?
+      primary?
+    else
+      specializations["traefik"]
+    end
+  end
+
+  def primary?
+    self == @config.primary_role
   end
 
 
   def uses_cord?
-    running_traefik? && cord_volume && health_check_cmd.present?
+    running_traefik? && cord_volume && healthcheck.cmd.present?
   end
 
   def cord_host_directory
-    File.join config.run_directory_as_docker_volume, "cords", [container_prefix, config.run_id].join("-")
+    File.join config.run_directory_as_docker_volume, "cords", [ container_prefix, config.run_id ].join("-")
   end
 
   def cord_volume
-    if (cord = health_check_options["cord"])
+    if (cord = healthcheck.cord)
       @cord_volume ||= Kamal::Configuration::Volume.new \
-        host_path: File.join(config.run_directory, "cords", [container_prefix, config.run_id].join("-")),
+        host_path: File.join(config.run_directory, "cords", [ container_prefix, config.run_id ].join("-")),
         container_path: cord
     end
   end
@@ -159,22 +207,37 @@ class Kamal::Configuration::Role
   end
 
   private
-    attr_accessor :config
+    def tagged_hosts
+      {}.tap do |tagged_hosts|
+        extract_hosts_from_config.map do |host_config|
+          if host_config.is_a?(Hash)
+            host, tags = host_config.first
+            tagged_hosts[host] = Array(tags)
+          elsif host_config.is_a?(String)
+            tagged_hosts[host_config] = []
+          end
+        end
+      end
+    end
 
     def extract_hosts_from_config
-      if config.servers.is_a?(Array)
+      if config.raw_config.servers.is_a?(Array)
-        config.servers
+        config.raw_config.servers
       else
-        servers = config.servers[name]
+        servers = config.raw_config.servers[name]
         servers.is_a?(Array) ? servers : Array(servers["hosts"])
       end
     end
 
     def default_labels
-      if config.destination
+      { "service" => config.service, "role" => name, "destination" => config.destination }
-        { "service" => config.service, "role" => name, "destination" => config.destination }
+    end
+
+    def specializations
+      if config.raw_config.servers.is_a?(Array) || config.raw_config.servers[name].is_a?(Array)
+        {}
       else
-        { "service" => config.service, "role" => name }
+        config.raw_config.servers[name]
       end
     end
 
@@ -196,7 +259,7 @@ class Kamal::Configuration::Role
     end
 
     def traefik_service
-      [ config.service, name, config.destination ].compact.join("-")
+      container_prefix
     end
 
     def custom_labels
@@ -205,45 +268,4 @@ class Kamal::Configuration::Role
         labels.merge!(specializations["labels"]) if specializations["labels"].present?
       end
     end
-
-    def specializations
-      if config.servers.is_a?(Array) || config.servers[name].is_a?(Array)
-        { }
-      else
-        config.servers[name].except("hosts")
-      end
-    end
-
-    def specialized_env
-      specializations["env"] || {}
-    end
-
-    def merged_env
-      config.env&.merge(specialized_env) || {}
-    end
-
-    # Secrets are stored in an array, which won't merge by default, so have to do it by hand.
-    def merged_env_with_secrets
-      merged_env.tap do |new_env|
-        new_env["secret"] = Array(config.env["secret"]) + Array(specialized_env["secret"])
-
-        # If there's no secret/clear split, everything is clear
-        clear_app_env  = config.env["secret"] ? Array(config.env["clear"]) : Array(config.env["clear"] || config.env)
-        clear_role_env = specialized_env["secret"] ? Array(specialized_env["clear"]) : Array(specialized_env["clear"] || specialized_env)
-
-        new_env["clear"] = (clear_app_env + clear_role_env).uniq
-      end
-    end
-
-    def http_health_check(port:, path:)
-      "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present?
-    end
-
-    def health_check_options
-      @health_check_options ||= begin
-        options = specializations["healthcheck"] || {}
-        options = config.healthcheck.merge(options) if running_traefik?
-        options
-      end
-    end
 end

lib/kamal/configuration/servers.rb

@@ -0,0 +1,18 @@
+class Kamal::Configuration::Servers
+  include Kamal::Configuration::Validation
+
+  attr_reader :config, :servers_config, :roles
+
+  def initialize(config:)
+    @config = config
+    @servers_config = config.raw_config.servers
+    validate! servers_config, with: Kamal::Configuration::Validator::Servers
+
+    @roles = role_names.map { |role_name| Kamal::Configuration::Role.new role_name, config: config }
+  end
+
+  private
+    def role_names
+      servers_config.is_a?(Array) ? [ "web" ] : servers_config.keys.sort
+    end
+end

lib/kamal/configuration/ssh.rb

@@ -1,24 +1,45 @@
 class Kamal::Configuration::Ssh
   LOGGER = ::Logger.new(STDERR)
 
+  include Kamal::Configuration::Validation
+
+  attr_reader :ssh_config
+
   def initialize(config:)
-    @config = config.raw_config.ssh || {}
+    @ssh_config = config.raw_config.ssh || {}
+    validate! ssh_config
   end
 
   def user
-    config.fetch("user", "root")
+    ssh_config.fetch("user", "root")
+  end
+
+  def port
+    ssh_config.fetch("port", 22)
   end
 
   def proxy
-    if (proxy = config["proxy"])
+    if (proxy = ssh_config["proxy"])
       Net::SSH::Proxy::Jump.new(proxy.include?("@") ? proxy : "root@#{proxy}")
-    elsif (proxy_command = config["proxy_command"])
+    elsif (proxy_command = ssh_config["proxy_command"])
       Net::SSH::Proxy::Command.new(proxy_command)
     end
   end
 
+  def keys_only
+    ssh_config["keys_only"]
+  end
+
+  def keys
+    ssh_config["keys"]
+  end
+
+  def key_data
+    ssh_config["key_data"]
+  end
+
   def options
-    { user: user, proxy: proxy, logger: logger, keepalive: true, keepalive_interval: 30 }.compact
+    { user: user, port: port, proxy: proxy, logger: logger, keepalive: true, keepalive_interval: 30, keys_only: keys_only, keys: keys, key_data: key_data }.compact
   end
 
   def to_h
@@ -26,13 +47,11 @@ class Kamal::Configuration::Ssh
   end
 
   private
-    attr_accessor :config
-
     def logger
       LOGGER.tap { |logger| logger.level = log_level }
     end
 
     def log_level
-      config.fetch("log_level", :fatal)
+      ssh_config.fetch("log_level", :fatal)
     end
 end

lib/kamal/configuration/sshkit.rb

@@ -1,20 +1,22 @@
 class Kamal::Configuration::Sshkit
+  include Kamal::Configuration::Validation
+
+  attr_reader :sshkit_config
+
   def initialize(config:)
-    @options = config.raw_config.sshkit || {}
+    @sshkit_config = config.raw_config.sshkit || {}
+    validate! sshkit_config
   end
 
   def max_concurrent_starts
-    options.fetch("max_concurrent_starts", 30)
+    sshkit_config.fetch("max_concurrent_starts", 30)
   end
 
   def pool_idle_timeout
-    options.fetch("pool_idle_timeout", 900)
+    sshkit_config.fetch("pool_idle_timeout", 900)
   end
 
   def to_h
-    options
+    sshkit_config
   end
-
-  private
-    attr_accessor :options
 end

lib/kamal/configuration/traefik.rb

@@ -0,0 +1,78 @@
+class Kamal::Configuration::Traefik
+  delegate :argumentize, to: Kamal::Utils
+
+  DEFAULT_IMAGE = "traefik:v2.10"
+  CONTAINER_PORT = 80
+  DEFAULT_ARGS = {
+    "log.level" => "DEBUG"
+  }
+  DEFAULT_LABELS = {
+    # These ensure we serve a 502 rather than a 404 if no containers are available
+    "traefik.http.routers.catchall.entryPoints" => "http",
+    "traefik.http.routers.catchall.rule" => "PathPrefix(`/`)",
+    "traefik.http.routers.catchall.service" => "unavailable",
+    "traefik.http.routers.catchall.priority" => 1,
+    "traefik.http.services.unavailable.loadbalancer.server.port" => "0"
+  }
+
+  include Kamal::Configuration::Validation
+
+  attr_reader :config, :traefik_config
+
+  def initialize(config:)
+    @config = config
+    @traefik_config = config.raw_config.traefik || {}
+    validate! traefik_config
+  end
+
+  def publish?
+    traefik_config["publish"] != false
+  end
+
+  def labels
+    DEFAULT_LABELS.merge(traefik_config["labels"] || {})
+  end
+
+  def env
+    Kamal::Configuration::Env.new \
+      config: traefik_config.fetch("env", {}),
+      secrets: config.secrets,
+      context: "traefik/env"
+  end
+
+  def host_port
+    traefik_config.fetch("host_port", CONTAINER_PORT)
+  end
+
+  def options
+    traefik_config.fetch("options", {})
+  end
+
+  def port
+    "#{host_port}:#{CONTAINER_PORT}"
+  end
+
+  def args
+    DEFAULT_ARGS.merge(traefik_config.fetch("args", {}))
+  end
+
+  def image
+    traefik_config.fetch("image", DEFAULT_IMAGE)
+  end
+
+  def env_args
+    [ *env.clear_args, *argumentize("--env-file", secrets_path) ]
+  end
+
+  def env_directory
+    File.join(config.env_directory, "traefik")
+  end
+
+  def secrets_io
+    env.secrets_io
+  end
+
+  def secrets_path
+    File.join(config.env_directory, "traefik", "traefik.env")
+  end
+end