Bump version for 1.1.0

Reset the env before pushing

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kamal (0.16.1)
+    kamal (1.1.0)
       activesupport (>= 7.0)
       bcrypt_pbkdf (~> 1.0)
       concurrent-ruby (~> 1.2)

README.md

@@ -1,6 +1,6 @@
 # Kamal: Deploy web apps anywhere
 
-From bare metal to cloud VMs using Docker, deploy web apps anywhere with zero downtime. Kamal uses the dynamic reverse-proxy Traefik to hold requests, while the new app container is started and the old one is stopped — working seamlessly across multiple hosts, using SSHKit to execute commands. Originally built for Rails apps, Kamal will work with any type of web app that can be containerized with Docker.
+From bare metal to cloud VMs, deploy web apps anywhere with zero downtime. Kamal has the dynamic reverse-proxy Traefik hold requests while a new app container is started and the old one is stopped. Works seamlessly across multiple hosts, using SSHKit to execute commands. Originally built for Rails apps, Kamal will work with any type of web app that can be containerized with Docker.
 
 ➡️ See [kamal-deploy.org](https://kamal-deploy.org) for documentation on [installation](https://kamal-deploy.org/docs/installation), [configuration](https://kamal-deploy.org/docs/configuration), and [commands](https://kamal-deploy.org/docs/commands).
 

lib/kamal/cli/app.rb

@@ -9,13 +9,22 @@ class Kamal::Cli::App < Kamal::Cli::Base
 
           on(KAMAL.hosts) do
             execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
-            execute *KAMAL.app.tag_current_as_latest
+            execute *KAMAL.app.tag_current_image_as_latest
+
+            KAMAL.roles_on(host).each do |role|
+              app = KAMAL.app(role: role)
+              role_config = KAMAL.config.role(role)
+
+              if role_config.assets?
+                execute *app.extract_assets
+                old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+                execute *app.sync_asset_volumes(old_version: old_version)
+              end
+            end
           end
 
           on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
-            roles = KAMAL.roles_on(host)
+            KAMAL.roles_on(host).each do |role|
-
-            roles.each do |role|
               app = KAMAL.app(role: role)
               auditor = KAMAL.auditor(role: role)
               role_config = KAMAL.config.role(role)
@@ -27,27 +36,28 @@ class Kamal::Cli::App < Kamal::Cli::Base
                 execute *app.rename_container(version: version, new_version: tmp_version)
               end
 
-              execute *auditor.record("Booted app version #{version}"), verbosity: :debug
-
               old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
 
-              if role_config.uses_cord?
+              execute *app.tie_cord(role_config.cord_host_file) if role_config.uses_cord?
-                execute *app.tie_cord(role_config.cord_host_file)
+
-              end
+              execute *auditor.record("Booted app version #{version}"), verbosity: :debug
 
               execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
 
-              Kamal::Utils::HealthcheckPoller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
+              Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
 
               if old_version.present?
                 if role_config.uses_cord?
                   cord = capture_with_info(*app.cord(version: old_version), raise_on_non_zero_exit: false).strip
                   if cord.present?
                     execute *app.cut_cord(cord)
-                    Kamal::Utils::HealthcheckPoller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
+                    Kamal::Cli::Healthcheck::Poller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
                   end
                 end
+
                 execute *app.stop(version: old_version), raise_on_non_zero_exit: false
+
+                execute *app.clean_up_assets if role_config.assets?
               end
             end
           end
@@ -105,14 +115,16 @@ class Kamal::Cli::App < Kamal::Cli::Base
       say "Get current version of running container...", :magenta unless options[:version]
       using_version(options[:version] || current_running_version) do |version|
         say "Launching interactive command with version #{version} via SSH from existing container on #{KAMAL.primary_host}...", :magenta
-        run_locally { exec KAMAL.app(role: "web").execute_in_existing_container_over_ssh(cmd, host: KAMAL.primary_host) }
+        run_locally { exec KAMAL.app(role: KAMAL.primary_role).execute_in_existing_container_over_ssh(cmd, host: KAMAL.primary_host) }
       end
 
     when options[:interactive]
       say "Get most recent version available as an image...", :magenta unless options[:version]
       using_version(version_or_latest) do |version|
         say "Launching interactive command with version #{version} via SSH from new container on #{KAMAL.primary_host}...", :magenta
-        run_locally { exec KAMAL.app(role: KAMAL.roles_on(KAMAL.primary_host).first).execute_in_new_container_over_ssh(cmd, host: KAMAL.primary_host) }
+        run_locally do
+          exec KAMAL.app(role: KAMAL.primary_role).execute_in_new_container_over_ssh(cmd, host: KAMAL.primary_host)
+        end
       end
 
     when options[:reuse]
@@ -135,8 +147,12 @@ class Kamal::Cli::App < Kamal::Cli::Base
       using_version(version_or_latest) do |version|
         say "Launching command with version #{version} from new container...", :magenta
         on(KAMAL.hosts) do |host|
+          roles = KAMAL.roles_on(host)
+
+          roles.each do |role|
             execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
-          puts_by_host host, capture_with_info(*KAMAL.app.execute_in_new_container(cmd))
+            puts_by_host host, capture_with_info(*KAMAL.app(role: role).execute_in_new_container(cmd))
+          end
         end
       end
     end

lib/kamal/cli/base.rb

@@ -24,6 +24,7 @@ module Kamal::Cli
 
     def initialize(*)
       super
+      @original_env = ENV.to_h.dup
       load_envs
       initialize_commander(options_with_subcommand_class_options)
     end
@@ -37,6 +38,12 @@ module Kamal::Cli
         end
       end
 
+      def reload_envs
+        ENV.clear
+        ENV.update(@original_env)
+        load_envs
+      end
+
       def options_with_subcommand_class_options
         options.merge(@_initializer.last[:class_options] || {})
       end
@@ -75,8 +82,6 @@ module Kamal::Cli
       def mutating
         return yield if KAMAL.holding_lock?
 
-        KAMAL.config.ensure_env_available
-
         run_hook "pre-connect"
 
         ensure_run_directory

lib/kamal/cli/build.rb

@@ -1,3 +1,5 @@
+require "uri"
+
 class Kamal::Cli::Build < Kamal::Cli::Base
   class BuildError < StandardError; end
 
@@ -17,7 +19,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base
       verify_local_dependencies
       run_hook "pre-build"
 
-      if (uncommitted_changes = Kamal::Utils.uncommitted_changes).present?
+      if (uncommitted_changes = Kamal::Git.uncommitted_changes).present?
         say "The following paths have uncommitted changes:\n #{uncommitted_changes}", :yellow
       end
 
@@ -48,6 +50,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base
         execute *KAMAL.auditor.record("Pulled image with version #{KAMAL.config.version}"), verbosity: :debug
         execute *KAMAL.builder.clean, raise_on_non_zero_exit: false
         execute *KAMAL.builder.pull
+        execute *KAMAL.builder.validate_image
       end
     end
   end
@@ -55,6 +58,10 @@ class Kamal::Cli::Build < Kamal::Cli::Base
   desc "create", "Create a build setup"
   def create
     mutating do
+      if (remote_host = KAMAL.config.builder.remote_host)
+        connect_to_remote_host(remote_host)
+      end
+
       run_locally do
         begin
           debug "Using builder: #{KAMAL.builder.name}"
@@ -103,4 +110,14 @@ class Kamal::Cli::Build < Kamal::Cli::Base
         end
       end
     end
+
+    def connect_to_remote_host(remote_host)
+      remote_uri = URI.parse(remote_host)
+      if remote_uri.scheme == "ssh"
+        options = { user: remote_uri.user, port: remote_uri.port }.compact
+        on(remote_uri.host, options) do
+          execute "true"
+        end
+      end
+    end
 end

lib/kamal/cli/env.rb

@@ -5,6 +5,8 @@ class Kamal::Cli::Env < Kamal::Cli::Base
   def push
     mutating do
       on(KAMAL.hosts) do
+        execute *KAMAL.auditor.record("Pushed env files"), verbosity: :debug
+
         KAMAL.roles_on(host).each do |role|
           role_config = KAMAL.config.role(role)
           execute *KAMAL.app(role: role).make_env_directory
@@ -31,6 +33,8 @@ class Kamal::Cli::Env < Kamal::Cli::Base
   def delete
     mutating do
       on(KAMAL.hosts) do
+        execute *KAMAL.auditor.record("Deleted env files"), verbosity: :debug
+
         KAMAL.roles_on(host).each do |role|
           role_config = KAMAL.config.role(role)
           execute *KAMAL.app(role: role).remove_env_file

lib/kamal/cli/healthcheck.rb

@@ -6,8 +6,8 @@ class Kamal::Cli::Healthcheck < Kamal::Cli::Base
     on(KAMAL.primary_host) do
       begin
         execute *KAMAL.healthcheck.run
-        Kamal::Utils::HealthcheckPoller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
+        Poller.wait_for_healthy { capture_with_info(*KAMAL.healthcheck.status) }
-      rescue Kamal::Utils::HealthcheckPoller::HealthcheckError => e
+      rescue Poller::HealthcheckError => e
         error capture_with_info(*KAMAL.healthcheck.logs)
         error capture_with_pretty_json(*KAMAL.healthcheck.container_health_log)
         raise

lib/kamal/cli/healthcheck/poller.rb

@@ -0,0 +1,64 @@
+module Kamal::Cli::Healthcheck::Poller
+  extend self
+
+  TRAEFIK_UPDATE_DELAY = 5
+
+  class HealthcheckError < StandardError; end
+
+  def wait_for_healthy(pause_after_ready: false, &block)
+    attempt = 1
+    max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+    begin
+      case status = block.call
+      when "healthy"
+        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
+      when "running" # No health check configured
+        sleep KAMAL.config.readiness_delay if pause_after_ready
+      else
+        raise HealthcheckError, "container not ready (#{status})"
+      end
+    rescue HealthcheckError => e
+      if attempt <= max_attempts
+        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+        sleep attempt
+        attempt += 1
+        retry
+      else
+        raise
+      end
+    end
+
+    info "Container is healthy!"
+  end
+
+  def wait_for_unhealthy(pause_after_ready: false, &block)
+    attempt = 1
+    max_attempts = KAMAL.config.healthcheck["max_attempts"]
+
+    begin
+      case status = block.call
+      when "unhealthy"
+        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
+      else
+        raise HealthcheckError, "container not unhealthy (#{status})"
+      end
+    rescue HealthcheckError => e
+      if attempt <= max_attempts
+        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+        sleep attempt
+        attempt += 1
+        retry
+      else
+        raise
+      end
+    end
+
+    info "Container is unhealthy!"
+  end
+
+  private
+    def info(message)
+      SSHKit.config.output.info(message)
+    end
+end

lib/kamal/cli/main.rb

@@ -1,9 +1,14 @@
 class Kamal::Cli::Main < Kamal::Cli::Base
-  desc "setup", "Setup all accessories and deploy app to servers"
+  desc "setup", "Setup all accessories, push the env, and deploy app to servers"
   def setup
     print_runtime do
       mutating do
+        say "Ensure Docker is installed...", :magenta
         invoke "kamal:cli:server:bootstrap"
+
+        say "Push env files...", :magenta
+        invoke "kamal:cli:env:push"
+
         invoke "kamal:cli:accessory:boot", [ "all" ]
         deploy
       end
@@ -37,7 +42,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
         invoke "kamal:cli:healthcheck:perform", [], invoke_options
 
         say "Detect stale containers...", :magenta
-        invoke "kamal:cli:app:stale_containers", [], invoke_options
+        invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
 
         invoke "kamal:cli:app:boot", [], invoke_options
 
@@ -70,7 +75,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
         invoke "kamal:cli:healthcheck:perform", [], invoke_options
 
         say "Detect stale containers...", :magenta
-        invoke "kamal:cli:app:stale_containers", [], invoke_options
+        invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
 
         invoke "kamal:cli:app:boot", [], invoke_options
       end
@@ -165,6 +170,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   end
 
   desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)"
+  option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip .env file push"
   def envify
     if destination = options[:destination]
       env_template_path = ".env.#{destination}.erb"
@@ -174,11 +180,13 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       env_path          = ".env"
     end
 
-    File.write(env_path, ERB.new(File.read(env_template_path)).result, perm: 0600)
+    File.write(env_path, ERB.new(File.read(env_template_path), trim_mode: "-").result, perm: 0600)
 
-    load_envs # reload new file
+    unless options[:skip_push]
+      reload_envs
       invoke "kamal:cli:env:push", options
     end
+  end
 
   desc "remove", "Remove Traefik, app, accessories, and registry session from servers"
   option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"

lib/kamal/cli/prune.rb

@@ -7,7 +7,7 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
     end
   end
 
-  desc "images", "Prune dangling images"
+  desc "images", "Prune unused images"
   def images
     mutating do
       on(KAMAL.hosts) do
@@ -23,7 +23,8 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
     mutating do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug
-        execute *KAMAL.prune.containers
+        execute *KAMAL.prune.app_containers
+        execute *KAMAL.prune.healthcheck_containers
       end
     end
   end

lib/kamal/cli/server.rb

@@ -12,9 +12,7 @@ class Kamal::Cli::Server < Kamal::Cli::Base
           missing << host
         end
       end
-    end
 
-    on(KAMAL.hosts) do
       execute(*KAMAL.server.ensure_run_directory)
     end
 

lib/kamal/cli/templates/deploy.yml

@@ -19,6 +19,7 @@ registry:
     - KAMAL_REGISTRY_PASSWORD
 
 # Inject ENV variables into containers (secrets come from .env).
+# Remember to run `kamal env push` after making changes!
 # env:
 #   clear:
 #     DB_HOST: 192.168.0.2
@@ -52,7 +53,7 @@ registry:
 #         - MYSQL_ROOT_PASSWORD
 #     files:
 #       - config/mysql/production.cnf:/etc/mysql/my.cnf
-#       - db/production.sql.erb:/docker-entrypoint-initdb.d/setup.sql
+#       - db/production.sql:/docker-entrypoint-initdb.d/setup.sql
 #     directories:
 #       - data:/var/lib/mysql
 #   redis:
@@ -72,3 +73,13 @@ registry:
 # healthcheck:
 #   path: /healthz
 #   port: 4000
+
+# Bridge fingerprinted assets, like JS and CSS, between versions to avoid
+# hitting 404 on in-flight requests. Combines all files from new and old
+# version inside the asset_path.
+# asset_path: /rails/public/assets
+
+# Configure rolling deploys by setting a wait time between batches of restarts.
+# boot:
+#   limit: 10 # Can also specify as a percentage of total hosts, such as "25%"
+#   wait: 2

lib/kamal/commander.rb

@@ -39,6 +39,10 @@ class Kamal::Commander
     specific_hosts&.first || specific_roles&.first&.primary_host || config.primary_web_host
   end
 
+  def primary_role
+    roles_on(primary_host).first
+  end
+
   def roles
     (specific_roles || config.roles).select do |role|
       ((specific_hosts || config.all_hosts) & role.hosts).any?
@@ -51,14 +55,6 @@ class Kamal::Commander
     end
   end
 
-  def boot_strategy
-    if config.boot.limit.present?
-      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
-    else
-      {}
-    end
-  end
-
   def roles_on(host)
     roles.select { |role| role.hosts.include?(host.to_s) }.map(&:name)
   end
@@ -128,6 +124,7 @@ class Kamal::Commander
     @traefik ||= Kamal::Commands::Traefik.new(config)
   end
 
+
   def with_verbosity(level)
     old_level = self.verbosity
 
@@ -140,6 +137,14 @@ class Kamal::Commander
     SSHKit.config.output_verbosity = old_level
   end
 
+  def boot_strategy
+    if config.boot.limit.present?
+      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
+    else
+      {}
+    end
+  end
+
   def holding_lock?
     self.holding_lock
   end

lib/kamal/commands/app.rb

@@ -1,4 +1,6 @@
 class Kamal::Commands::App < Kamal::Commands::Base
+  include Assets, Containers, Cord, Execution, Images, Logging
+
   ACTIVE_DOCKER_STATUSES = [ :running, :restarting ]
 
   attr_reader :role, :role_config
@@ -16,10 +18,12 @@ class Kamal::Commands::App < Kamal::Commands::Base
       "--name", container_name,
       *(["--hostname", hostname] if hostname),
       "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
+      "-e", "KAMAL_VERSION=\"#{config.version}\"",
       *role_config.env_args,
       *role_config.health_check_args,
       *config.logging_args,
       *config.volume_args,
+      *role_config.asset_volume_args,
       *role_config.label_args,
       *role_config.option_args,
       config.absolute_image,
@@ -45,51 +49,6 @@ class Kamal::Commands::App < Kamal::Commands::Base
   end
 
 
-  def logs(since: nil, lines: nil, grep: nil)
-    pipe \
-      current_running_container_id,
-      "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
-      ("grep '#{grep}'" if grep)
-  end
-
-  def follow_logs(host:, grep: nil)
-    run_over_ssh \
-      pipe(
-        current_running_container_id,
-        "xargs docker logs --timestamps --tail 10 --follow 2>&1",
-        (%(grep "#{grep}") if grep)
-      ),
-      host: host
-  end
-
-
-  def execute_in_existing_container(*command, interactive: false)
-    docker :exec,
-      ("-it" if interactive),
-      container_name,
-      *command
-  end
-
-  def execute_in_new_container(*command, interactive: false)
-    docker :run,
-      ("-it" if interactive),
-      "--rm",
-      *role_config&.env_args,
-      *config.volume_args,
-      *role_config&.option_args,
-      config.absolute_image,
-      *command
-  end
-
-  def execute_in_existing_container_over_ssh(*command, host:)
-    run_over_ssh execute_in_existing_container(*command, interactive: true), host: host
-  end
-
-  def execute_in_new_container_over_ssh(*command, host:)
-    run_over_ssh execute_in_new_container(*command, interactive: true), host: host
-  end
-
-
   def current_running_container_id
     docker :ps, "--quiet", *filter_args(statuses: ACTIVE_DOCKER_STATUSES), "--latest"
   end
@@ -105,42 +64,9 @@ class Kamal::Commands::App < Kamal::Commands::Base
   def list_versions(*docker_args, statuses: nil)
     pipe \
       docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
-      %(while read line; do echo ${line##{role_config.full_name}-}; done) # Extract SHA from "service-role-dest-SHA"
+      %(while read line; do echo ${line##{role_config.container_prefix}-}; done) # Extract SHA from "service-role-dest-SHA"
   end
 
-  def list_containers
-    docker :container, :ls, "--all", *filter_args
-  end
-
-  def list_container_names
-    [ *list_containers, "--format", "'{{ .Names }}'" ]
-  end
-
-  def remove_container(version:)
-    pipe \
-      container_id_for(container_name: container_name(version)),
-      xargs(docker(:container, :rm))
-  end
-
-  def rename_container(version:, new_version:)
-    docker :rename, container_name(version), container_name(new_version)
-  end
-
-  def remove_containers
-    docker :container, :prune, "--force", *filter_args
-  end
-
-  def list_images
-    docker :image, :ls, config.repository
-  end
-
-  def remove_images
-    docker :image, :prune, "--all", "--force", *filter_args
-  end
-
-  def tag_current_as_latest
-    docker :tag, config.absolute_image, config.latest_image
-  end
 
   def make_env_directory
     make_directory role_config.host_env_directory
@@ -150,23 +76,10 @@ class Kamal::Commands::App < Kamal::Commands::Base
     [ :rm, "-f", role_config.host_env_file_path ]
   end
 
-  def cord(version:)
-    pipe \
-      docker(:inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", container_name(version)),
-      [:awk, "'$2 == \"#{role_config.cord_container_directory}\" {print $1}'"]
-  end
-
-  def tie_cord(cord)
-    create_empty_file(cord)
-  end
-
-  def cut_cord(cord)
-    remove_directory(cord)
-  end
 
   private
     def container_name(version = nil)
-      [ role_config.full_name, version || config.version ].compact.join("-")
+      [ role_config.container_prefix, version || config.version ].compact.join("-")
     end
 
     def filter_args(statuses: nil)

lib/kamal/commands/app/assets.rb

@@ -0,0 +1,51 @@
+module Kamal::Commands::App::Assets
+  def extract_assets
+    asset_container = "#{role_config.container_prefix}-assets"
+
+    combine \
+      make_directory(role_config.asset_extracted_path),
+      [*docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true"],
+      docker(:run, "--name", asset_container, "--detach", "--rm", config.latest_image, "sleep 1000000"),
+      docker(:cp, "-L", "#{asset_container}:#{role_config.asset_path}/.", role_config.asset_extracted_path),
+      docker(:stop, "-t 1", asset_container),
+      by: "&&"
+  end
+
+  def sync_asset_volumes(old_version: nil)
+    new_extracted_path, new_volume_path = role_config.asset_extracted_path(config.version), role_config.asset_volume.host_path
+    if old_version.present?
+      old_extracted_path, old_volume_path = role_config.asset_extracted_path(old_version), role_config.asset_volume(old_version).host_path
+    end
+
+    commands = [make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path)]
+
+    if old_version.present?
+      commands << copy_contents(new_extracted_path, old_volume_path, continue_on_error: true)
+      commands << copy_contents(old_extracted_path, new_volume_path, continue_on_error: true)
+    end
+
+    chain *commands
+  end
+
+  def clean_up_assets
+    chain \
+      find_and_remove_older_siblings(role_config.asset_extracted_path),
+      find_and_remove_older_siblings(role_config.asset_volume_path)
+  end
+
+  private
+    def find_and_remove_older_siblings(path)
+      [
+        :find,
+        Pathname.new(path).dirname.to_s,
+        "-maxdepth 1",
+        "-name", "'#{role_config.container_prefix}-*'",
+        "!", "-name", Pathname.new(path).basename.to_s,
+        "-exec rm -rf \"{}\" +"
+      ]
+    end
+
+    def copy_contents(source, destination, continue_on_error: false)
+      [ :cp, "-rnT", "#{source}", destination, *("|| true" if continue_on_error)]
+    end
+end

lib/kamal/commands/app/containers.rb

@@ -0,0 +1,23 @@
+module Kamal::Commands::App::Containers
+  def list_containers
+    docker :container, :ls, "--all", *filter_args
+  end
+
+  def list_container_names
+    [ *list_containers, "--format", "'{{ .Names }}'" ]
+  end
+
+  def remove_container(version:)
+    pipe \
+      container_id_for(container_name: container_name(version)),
+      xargs(docker(:container, :rm))
+  end
+
+  def rename_container(version:, new_version:)
+    docker :rename, container_name(version), container_name(new_version)
+  end
+
+  def remove_containers
+    docker :container, :prune, "--force", *filter_args
+  end
+end

lib/kamal/commands/app/cord.rb

@@ -0,0 +1,22 @@
+module Kamal::Commands::App::Cord
+  def cord(version:)
+    pipe \
+      docker(:inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", container_name(version)),
+      [:awk, "'$2 == \"#{role_config.cord_volume.container_path}\" {print $1}'"]
+  end
+
+  def tie_cord(cord)
+    create_empty_file(cord)
+  end
+
+  def cut_cord(cord)
+    remove_directory(cord)
+  end
+  
+  private    
+    def create_empty_file(file)
+      chain \
+        make_directory_for(file),
+        [:touch, file]
+    end
+end

lib/kamal/commands/app/execution.rb

@@ -0,0 +1,27 @@
+module Kamal::Commands::App::Execution
+  def execute_in_existing_container(*command, interactive: false)
+    docker :exec,
+      ("-it" if interactive),
+      container_name,
+      *command
+  end
+
+  def execute_in_new_container(*command, interactive: false)
+    docker :run,
+      ("-it" if interactive),
+      "--rm",
+      *role_config&.env_args,
+      *config.volume_args,
+      *role_config&.option_args,
+      config.absolute_image,
+      *command
+  end
+
+  def execute_in_existing_container_over_ssh(*command, host:)
+    run_over_ssh execute_in_existing_container(*command, interactive: true), host: host
+  end
+
+  def execute_in_new_container_over_ssh(*command, host:)
+    run_over_ssh execute_in_new_container(*command, interactive: true), host: host
+  end
+end

lib/kamal/commands/app/images.rb

@@ -0,0 +1,13 @@
+module Kamal::Commands::App::Images
+  def list_images
+    docker :image, :ls, config.repository
+  end
+
+  def remove_images
+    docker :image, :prune, "--all", "--force", *filter_args
+  end
+
+  def tag_current_image_as_latest
+    docker :tag, config.absolute_image, config.latest_image
+  end
+end

lib/kamal/commands/app/logging.rb

@@ -0,0 +1,18 @@
+module Kamal::Commands::App::Logging
+  def logs(since: nil, lines: nil, grep: nil)
+    pipe \
+      current_running_container_id,
+      "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
+      ("grep '#{grep}'" if grep)
+  end
+
+  def follow_logs(host:, grep: nil)
+    run_over_ssh \
+      pipe(
+        current_running_container_id,
+        "xargs docker logs --timestamps --tail 10 --follow 2>&1",
+        (%(grep "#{grep}") if grep)
+      ),
+      host: host
+  end
+end

lib/kamal/commands/base.rb

@@ -73,11 +73,5 @@ module Kamal::Commands
       def tags(**details)
         Kamal::Tags.from_config(config, **details)
       end
-
-      def create_empty_file(file)
-        chain \
-          make_directory_for(file),
-          [:touch, file]
-      end
   end
 end

lib/kamal/commands/builder.rb

@@ -1,7 +1,7 @@
 require "active_support/core_ext/string/filters"
 
 class Kamal::Commands::Builder < Kamal::Commands::Base
-  delegate :create, :remove, :push, :clean, :pull, :info, to: :target
+  delegate :create, :remove, :push, :clean, :pull, :info, :validate_image, to: :target
 
   def name
     target.class.to_s.remove("Kamal::Commands::Builder::").underscore.inquiry

lib/kamal/commands/builder/base.rb

@@ -21,6 +21,12 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
     config.builder.context
   end
 
+  def validate_image
+    pipe \
+      docker(:inspect, "-f", "'{{ .Config.Labels.service }}'", config.absolute_image),
+      [:grep, "-x", config.service, "||", "(echo \"Image #{config.absolute_image} is missing the `service` label\" && exit 1)"]
+  end
+
 
   private
     def build_tags

lib/kamal/commands/docker.rb

@@ -16,6 +16,6 @@ class Kamal::Commands::Docker < Kamal::Commands::Base
 
   # Do we have superuser access to install Docker and start system services?
   def superuser?
-    [ '[ "${EUID:-$(id -u)}" -eq 0 ]' ]
+    [ '[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null' ]
   end
 end

lib/kamal/commands/healthcheck.rb

@@ -7,8 +7,8 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
       "--detach",
       "--name", container_name_with_version,
       "--publish", "#{exposed_port}:#{config.healthcheck["port"]}",
-      "--label", "service=#{container_name}",
+      "--label", "service=#{config.healthcheck_service}",
-      "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"",
+      "-e", "KAMAL_CONTAINER_NAME=\"#{config.healthcheck_service}\"",
       *web.env_args,
       *web.health_check_args(cord: false),
       *config.volume_args,
@@ -26,7 +26,7 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
   end
 
   def logs
-    pipe container_id, xargs(docker(:logs, "--tail", 50, "2>&1"))
+    pipe container_id, xargs(docker(:logs, "--tail", log_lines, "2>&1"))
   end
 
   def stop
@@ -38,12 +38,8 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
   end
 
   private
-    def container_name
-      [ "healthcheck", config.service, config.destination ].compact.join("-")
-    end
-
     def container_name_with_version
-      "#{container_name}-#{config.version}"
+      "#{config.healthcheck_service}-#{config.version}"
     end
 
     def container_id
@@ -57,4 +53,8 @@ class Kamal::Commands::Healthcheck < Kamal::Commands::Base
     def exposed_port
       config.healthcheck["exposed_port"]
     end
+
+    def log_lines
+      config.healthcheck["log_lines"]
+    end
 end

lib/kamal/commands/lock.rb

@@ -56,7 +56,7 @@ class Kamal::Commands::Lock < Kamal::Commands::Base
     end
 
     def locked_by
-      `git config user.name`.strip
+      Kamal::Git.user_name
     rescue Errno::ENOENT
       "Unknown"
     end

lib/kamal/commands/prune.rb

@@ -3,7 +3,7 @@ require "active_support/core_ext/numeric/time"
 
 class Kamal::Commands::Prune < Kamal::Commands::Base
   def dangling_images
-    docker :image, :prune, "--force", "--filter", "label=service=#{config.service}", "--filter", "dangling=true"
+    docker :image, :prune, "--force", "--filter", "label=service=#{config.service}"
   end
 
   def tagged_images
@@ -13,13 +13,17 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
       "while read image tag; do docker rmi $tag; done"
   end
 
-  def containers(keep_last: 5)
+  def app_containers(keep_last: 5)
     pipe \
       docker(:ps, "-q", "-a", *service_filter, *stopped_containers_filters),
       "tail -n +#{keep_last + 1}",
       "while read container_id; do docker rm $container_id; done"
   end
 
+  def healthcheck_containers
+    docker :container, :prune, "--force", *healthcheck_service_filter
+  end
+
   private
     def stopped_containers_filters
       [ "created", "exited", "dead" ].flat_map { |status| ["--filter", "status=#{status}"] }
@@ -35,4 +39,8 @@ class Kamal::Commands::Prune < Kamal::Commands::Base
     def service_filter
       [ "--filter", "label=service=#{config.service}" ]
     end
+
+    def healthcheck_service_filter
+      [ "--filter", "label=service=#{config.healthcheck_service}" ]
+    end
   end

lib/kamal/commands/traefik.rb

@@ -1,5 +1,5 @@
 class Kamal::Commands::Traefik < Kamal::Commands::Base
-  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
   DEFAULT_IMAGE = "traefik:v2.9"
   CONTAINER_PORT = 80
@@ -64,7 +64,7 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
   end
 
   def env_file
-    env_file_with_secrets config.traefik.fetch("env", {})
+    Kamal::EnvFile.new(config.traefik.fetch("env", {}))
   end
 
   def host_env_file_path

lib/kamal/configuration.rb

@@ -9,8 +9,7 @@ class Kamal::Configuration
   delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
   delegate :argumentize, :optionize, to: Kamal::Utils
 
-  attr_accessor :destination
+  attr_reader :destination, :raw_config
-  attr_accessor :raw_config
 
   class << self
     def create_from(config_file:, destination: nil, version: nil)
@@ -54,20 +53,19 @@ class Kamal::Configuration
   end
 
   def abbreviated_version
-    Kamal::Utils.abbreviate_version(version)
+    if version
-  end
+      # Don't abbreviate <sha>_uncommitted_<etc>
-
+      if version.include?("_")
-  def run_directory
+        version
-    raw_config.run_directory || ".kamal"
-  end
-
-  def run_directory_as_docker_volume
-    if Pathname.new(run_directory).absolute?
-      run_directory
       else
-      File.join "$(pwd)", run_directory
+        version[0...7]
       end
     end
+  end
+
+  def minimum_version
+    raw_config.minimum_version
+  end
 
 
   def roles
@@ -99,10 +97,6 @@ class Kamal::Configuration
     roles.select(&:running_traefik?).flat_map(&:hosts).uniq
   end
 
-  def boot
-    Kamal::Configuration::Boot.new(config: self)
-  end
-
 
   def repository
     [ raw_config.registry["server"], image ].compact.join("/")
@@ -120,6 +114,10 @@ class Kamal::Configuration
     "#{service}-#{version}"
   end
 
+  def require_destination?
+    raw_config.require_destination
+  end
+
 
   def volume_args
     if raw_config.volumes.present?
@@ -139,6 +137,18 @@ class Kamal::Configuration
   end
 
 
+  def boot
+    Kamal::Configuration::Boot.new(config: self)
+  end
+
+  def builder
+    Kamal::Configuration::Builder.new(config: self)
+  end
+
+  def traefik
+    raw_config.traefik || {}
+  end
+
   def ssh
     Kamal::Configuration::Ssh.new(config: self)
   end
@@ -149,22 +159,51 @@ class Kamal::Configuration
 
 
   def healthcheck
-    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }.merge(raw_config.healthcheck || {})
+    { "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }.merge(raw_config.healthcheck || {})
+  end
+
+  def healthcheck_service
+    [ "healthcheck", service, destination ].compact.join("-")
   end
 
   def readiness_delay
     raw_config.readiness_delay || 7
   end
 
-  def minimum_version
+  def run_id
-    raw_config.minimum_version
+    @run_id ||= SecureRandom.hex(16)
   end
 
+
+  def run_directory
+    raw_config.run_directory || ".kamal"
+  end
+
+  def run_directory_as_docker_volume
+    if Pathname.new(run_directory).absolute?
+      run_directory
+    else
+      File.join "$(pwd)", run_directory
+    end
+  end
+
+  def hooks_path
+    raw_config.hooks_path || ".kamal/hooks"
+  end
+
+  def host_env_directory
+    "#{run_directory}/env"
+  end
+
+  def asset_path
+    raw_config.asset_path
+  end
+
+
   def valid?
-    ensure_required_keys_present && ensure_valid_kamal_version
+    ensure_destination_if_required && ensure_required_keys_present && ensure_valid_kamal_version
   end
 
-
   def to_h
     {
       roles: role_names,
@@ -184,35 +223,17 @@ class Kamal::Configuration
     }.compact
   end
 
-  def traefik
-    raw_config.traefik || {}
-  end
 
-  def hooks_path
+  private
-    raw_config.hooks_path || ".kamal/hooks"
+    # Will raise ArgumentError if any required config keys are missing
+    def ensure_destination_if_required
+      if require_destination? && destination.nil?
+        raise ArgumentError, "You must specify a destination"
       end
 
-  def builder
-    Kamal::Configuration::Builder.new(config: self)
-  end
-
-  # Will raise KeyError if any secret ENVs are missing
-  def ensure_env_available
-    roles.each(&:env_file)
-
       true
     end
 
-  def host_env_directory
-    "#{run_directory}/env"
-  end
-
-  def run_id
-    @run_id ||= SecureRandom.hex(16)
-  end
-
-  private
-    # Will raise ArgumentError if any required config keys are missing
     def ensure_required_keys_present
       %i[ service image registry servers ].each do |key|
         raise ArgumentError, "Missing required configuration for #{key}" unless raw_config[key].present?
@@ -250,10 +271,8 @@ class Kamal::Configuration
 
     def git_version
       @git_version ||=
-        if system("git rev-parse")
+        if Kamal::Git.used?
-          uncommitted_suffix = Kamal::Utils.uncommitted_changes.present? ? "_uncommitted_#{SecureRandom.hex(8)}" : ""
+          [ Kamal::Git.revision, Kamal::Git.uncommitted_changes.present? ? "_uncommitted_#{SecureRandom.hex(8)}" : "" ].join
-
-          "#{`git rev-parse HEAD`.strip}#{uncommitted_suffix}"
         else
           raise "Can't use commit hash as version, no git repository found in #{Dir.pwd}"
         end

lib/kamal/configuration/accessory.rb

@@ -1,5 +1,5 @@
 class Kamal::Configuration::Accessory
-  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_accessor :name, :specifics
 
@@ -46,7 +46,7 @@ class Kamal::Configuration::Accessory
   end
 
   def env_file
-    env_file_with_secrets env
+    Kamal::EnvFile.new(env)
   end
 
   def host_env_directory

lib/kamal/configuration/role.rb

@@ -1,6 +1,6 @@
 class Kamal::Configuration::Role
   CORD_FILE = "cord"
-  delegate :argumentize, :env_file_with_secrets, :optionize, to: Kamal::Utils
+  delegate :argumentize, :optionize, to: Kamal::Utils
 
   attr_accessor :name
 
@@ -16,6 +16,18 @@ class Kamal::Configuration::Role
     @hosts ||= extract_hosts_from_config
   end
 
+  def cmd
+    specializations["cmd"]
+  end
+
+  def option_args
+    if args = specializations["options"]
+      optionize args
+    else
+      []
+    end
+  end
+
   def labels
     default_labels.merge(traefik_labels).merge(custom_labels)
   end
@@ -24,6 +36,7 @@ class Kamal::Configuration::Role
     argumentize "--label", labels
   end
 
+
   def env
     if config.env && config.env["secret"]
       merged_env_with_secrets
@@ -33,7 +46,7 @@ class Kamal::Configuration::Role
   end
 
   def env_file
-    env_file_with_secrets env
+    Kamal::EnvFile.new(env)
   end
 
   def host_env_directory
@@ -48,11 +61,16 @@ class Kamal::Configuration::Role
     argumentize "--env-file", host_env_file_path
   end
 
+  def asset_volume_args
+    asset_volume&.docker_args
+  end
+
+
   def health_check_args(cord: true)
     if health_check_cmd.present?
       if cord && uses_cord?
         optionize({ "health-cmd" => health_check_cmd_with_cord, "health-interval" => health_check_interval })
-          .concat(["--volume", "#{cord_host_directory}:#{cord_container_directory}"])
+          .concat(cord_volume.docker_args)
       else
         optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval })
       end
@@ -73,16 +91,30 @@ class Kamal::Configuration::Role
     health_check_options["interval"] || "1s"
   end
 
+
+  def running_traefik?
+    name.web? || specializations["traefik"]
+  end
+
+
   def uses_cord?
-    running_traefik? && cord_container_directory.present? && health_check_cmd.present?
+    running_traefik? && cord_volume && health_check_cmd.present?
   end
 
   def cord_host_directory
-    File.join config.run_directory_as_docker_volume, "cords", [full_name, config.run_id].join("-")
+    File.join config.run_directory_as_docker_volume, "cords", [container_prefix, config.run_id].join("-")
+  end
+
+  def cord_volume
+    if (cord = health_check_options["cord"])
+      @cord_volume ||= Kamal::Configuration::Volume.new \
+        host_path: File.join(config.run_directory, "cords", [container_prefix, config.run_id].join("-")),
+        container_path: cord
+    end
   end
 
   def cord_host_file
-    File.join cord_host_directory, CORD_FILE
+    File.join cord_volume.host_path, CORD_FILE
   end
 
   def cord_container_directory
@@ -90,30 +122,42 @@ class Kamal::Configuration::Role
   end
 
   def cord_container_file
-    File.join cord_container_directory, CORD_FILE
+    File.join cord_volume.container_path, CORD_FILE
   end
 
 
-  def cmd
+  def container_name(version = nil)
-    specializations["cmd"]
+    [ container_prefix, version || config.version ].compact.join("-")
   end
 
-  def option_args
+  def container_prefix
-    if args = specializations["options"]
-      optionize args
-    else
-      []
-    end
-  end
-
-  def running_traefik?
-    name.web? || specializations["traefik"]
-  end
-
-  def full_name
     [ config.service, name, config.destination ].compact.join("-")
   end
 
+
+  def asset_path
+    specializations["asset_path"] || config.asset_path
+  end
+
+  def assets?
+    asset_path.present? && running_traefik?
+  end
+
+  def asset_volume(version = nil)
+    if assets?
+      Kamal::Configuration::Volume.new \
+        host_path: asset_volume_path(version), container_path: asset_path
+    end
+  end
+
+  def asset_extracted_path(version = nil)
+    File.join config.run_directory, "assets", "extracted", container_name(version)
+  end
+
+  def asset_volume_path(version = nil)
+    File.join config.run_directory, "assets", "volumes", container_name(version)
+  end
+
   private
     attr_accessor :config
 

lib/kamal/configuration/ssh.rb

@@ -18,7 +18,7 @@ class Kamal::Configuration::Ssh
   end
 
   def options
-    { user: user, proxy: proxy, auth_methods: [ "publickey" ], logger: logger, keepalive: true, keepalive_interval: 30 }.compact
+    { user: user, proxy: proxy, logger: logger, keepalive: true, keepalive_interval: 30 }.compact
   end
 
   def to_h

lib/kamal/configuration/volume.rb

@@ -0,0 +1,22 @@
+class Kamal::Configuration::Volume
+  attr_reader :host_path, :container_path
+  delegate :argumentize, to: Kamal::Utils
+
+  def initialize(host_path:, container_path:)
+    @host_path = host_path
+    @container_path = container_path
+  end
+
+  def docker_args
+    argumentize "--volume", "#{host_path_for_docker_volume}:#{container_path}"
+  end
+
+  private
+    def host_path_for_docker_volume
+      if Pathname.new(host_path).absolute?
+        host_path
+      else
+        File.join "$(pwd)", host_path
+      end
+    end
+end

lib/kamal/env_file.rb

@@ -0,0 +1,41 @@
+# Encode an env hash as a string where secret values have been looked up and all values escaped for Docker.
+class Kamal::EnvFile
+  def initialize(env)
+    @env = env
+  end
+  
+  def to_s
+    env_file = StringIO.new.tap do |contents|
+      if (secrets = @env["secret"]).present?
+        @env.fetch("secret", @env)&.each do |key|
+          contents << docker_env_file_line(key, ENV.fetch(key))
+        end
+
+        @env["clear"]&.each do |key, value|
+          contents << docker_env_file_line(key, value)
+        end
+      else
+        @env.fetch("clear", @env)&.each do |key, value|
+          contents << docker_env_file_line(key, value)
+        end
+      end
+    end.string
+
+    # Ensure the file has some contents to avoid the SSHKIT empty file warning
+    env_file.presence || "\n"
+  end
+
+  alias to_str to_s
+  
+  private
+    def docker_env_file_line(key, value)
+      "#{key.to_s}=#{escape_docker_env_file_value(value)}\n"
+    end
+
+    # Escape a value to make it safe to dump in a docker file.
+    def escape_docker_env_file_value(value)
+      # Doublequotes are treated literally in docker env files
+      # so remove leading and trailing ones and unescape any others
+      value.to_s.dump[1..-2].gsub(/\\"/, "\"")
+    end
+end

lib/kamal/git.rb

@@ -0,0 +1,19 @@
+module Kamal::Git
+  extend self
+
+  def used?
+    system("git rev-parse")
+  end
+
+  def user_name
+    `git config user.name`.strip
+  end
+
+  def revision
+    `git rev-parse HEAD`.strip
+  end
+
+  def uncommitted_changes
+    `git status --porcelain`.strip
+  end
+end

lib/kamal/utils.rb

@@ -16,26 +16,6 @@ module Kamal::Utils
     end
   end
 
-  def env_file_with_secrets(env)
-    env_file = StringIO.new.tap do |contents|
-      if (secrets = env["secret"]).present?
-        env.fetch("secret", env)&.each do |key|
-          contents << docker_env_file_line(key, ENV.fetch(key))
-        end
-        env["clear"]&.each do |key, value|
-          contents << docker_env_file_line(key, value)
-        end
-      else
-        env.fetch("clear", env)&.each do |key, value|
-          contents << docker_env_file_line(key, value)
-        end
-      end
-    end.string
-
-    # Ensure the file has some contents to avoid the SSHKIT empty file warning
-    env_file || "\n"
-  end
-
   # Returns a list of shell-dashed option arguments. If the value is true, it's treated like a value-less option.
   def optionize(args, with: nil)
     options = if with
@@ -72,47 +52,10 @@ module Kamal::Utils
     end
   end
 
-  def unredacted(value)
-    case
-    when value.respond_to?(:unredacted)
-      value.unredacted
-    when value.respond_to?(:transform_values)
-      value.transform_values { |value| unredacted value }
-    when value.respond_to?(:map)
-      value.map { |element| unredacted element }
-    else
-      value
-    end
-  end
-
   # Escape a value to make it safe for shell use.
   def escape_shell_value(value)
     value.to_s.dump
       .gsub(/`/, '\\\\`')
       .gsub(DOLLAR_SIGN_WITHOUT_SHELL_EXPANSION_REGEX, '\$')
   end
-
-  # Abbreviate a git revhash for concise display
-  def abbreviate_version(version)
-    if version
-      # Don't abbreviate <sha>_uncommitted_<etc>
-      if version.include?("_")
-        version
-      else
-        version[0...7]
-      end
-    end
-  end
-
-  def uncommitted_changes
-    `git status --porcelain`.strip
-  end
-
-  def docker_env_file_line(key, value)
-    if key.include?("\n") || value.to_s.include?("\n")
-      raise ArgumentError, "docker env file format does not support newlines in keys or values, key: #{key}"
-    end
-
-    "#{key.to_s}=#{value.to_s}\n"
-  end
 end

lib/kamal/utils/healthcheck_poller.rb

@@ -1,64 +0,0 @@
-class Kamal::Utils::HealthcheckPoller
-  TRAEFIK_UPDATE_DELAY = 2
-
-  class HealthcheckError < StandardError; end
-
-  class << self
-    def wait_for_healthy(pause_after_ready: false, &block)
-      attempt = 1
-      max_attempts = KAMAL.config.healthcheck["max_attempts"]
-
-      begin
-        case status = block.call
-        when "healthy"
-          sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
-        when "running" # No health check configured
-          sleep KAMAL.config.readiness_delay if pause_after_ready
-        else
-          raise HealthcheckError, "container not ready (#{status})"
-        end
-      rescue HealthcheckError => e
-        if attempt <= max_attempts
-          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-          sleep attempt
-          attempt += 1
-          retry
-        else
-          raise
-        end
-      end
-
-      info "Container is healthy!"
-    end
-
-    def wait_for_unhealthy(pause_after_ready: false, &block)
-      attempt = 1
-      max_attempts = KAMAL.config.healthcheck["max_attempts"]
-
-      begin
-        case status = block.call
-        when "unhealthy"
-          sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
-        else
-          raise HealthcheckError, "container not unhealthy (#{status})"
-        end
-      rescue HealthcheckError => e
-        if attempt <= max_attempts
-          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-          sleep attempt
-          attempt += 1
-          retry
-        else
-          raise
-        end
-      end
-
-      info "Container is unhealthy!"
-    end
-
-    private
-      def info(message)
-        SSHKit.config.output.info(message)
-      end
-  end
-end

lib/kamal/utils/sensitive.rb

@@ -1,4 +1,5 @@
 require "active_support/core_ext/module/delegation"
+require "sshkit"
 
 class Kamal::Utils::Sensitive
   # So SSHKit knows to redact these values.

lib/kamal/version.rb

@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "0.16.1"
+  VERSION = "1.1.0"
 end

test/cli/app_test.rb

@@ -27,7 +27,7 @@ class CliAppTest < CliTestCase
       .returns("123") # old version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
       .returns("cordfile") # old version
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -55,8 +55,6 @@ class CliAppTest < CliTestCase
   end
 
   test "boot errors leave lock in place" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
-
     Kamal::Cli::App.any_instance.expects(:using_version).raises(RuntimeError)
 
     assert !KAMAL.holding_lock?
@@ -66,6 +64,34 @@ class CliAppTest < CliTestCase
     assert KAMAL.holding_lock?
   end
 
+  test "boot with assets" do
+    Object.any_instance.stubs(:sleep)
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678") # running version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("123").twice # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .returns("") # old version
+
+    run_command("boot", config: :with_assets).tap do |output|
+      assert_match "docker tag dhh/app:latest dhh/app:latest", output
+      assert_match "/usr/bin/env mkdir -p .kamal/assets/volumes/app-web-latest ; cp -rnT .kamal/assets/extracted/app-web-latest .kamal/assets/volumes/app-web-latest ; cp -rnT .kamal/assets/extracted/app-web-latest .kamal/assets/volumes/app-web-123 || true ; cp -rnT .kamal/assets/extracted/app-web-123 .kamal/assets/volumes/app-web-latest || true", output
+      assert_match "/usr/bin/env mkdir -p .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets 2> /dev/null || true && docker run --name app-web-assets --detach --rm dhh/app:latest sleep 1000000 && docker cp -L app-web-assets:/public/assets/. .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets", output
+      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} /, output
+      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+      assert_match "/usr/bin/env find .kamal/assets/extracted -maxdepth 1 -name 'app-web-*' ! -name app-web-latest -exec rm -rf \"{}\" + ; find .kamal/assets/volumes -maxdepth 1 -name 'app-web-*' ! -name app-web-latest -exec rm -rf \"{}\" +", output
+    end
+  end
+
   test "start" do
     run_command("start").tap do |output|
       assert_match "docker start app-web-999", output
@@ -133,7 +159,7 @@ class CliAppTest < CliTestCase
 
   test "exec" do
     run_command("exec", "ruby -v").tap do |output|
-      assert_match "docker run --rm dhh/app:latest ruby -v", output
+      assert_match "docker run --rm --env-file .kamal/env/roles/app-web.env dhh/app:latest ruby -v", output
     end
   end
 
@@ -144,6 +170,25 @@ class CliAppTest < CliTestCase
     end
   end
 
+  test "exec interactive" do
+    SSHKit::Backend::Abstract.any_instance.expects(:exec)
+      .with("ssh -t root@1.1.1.1 'docker run -it --rm --env-file .kamal/env/roles/app-web.env dhh/app:latest ruby -v'")
+    run_command("exec", "-i", "ruby -v").tap do |output|
+      assert_match "Get most recent version available as an image...", output
+      assert_match "Launching interactive command with version latest via SSH from new container on 1.1.1.1...", output
+    end
+  end
+
+  test "exec interactive with reuse" do
+    SSHKit::Backend::Abstract.any_instance.expects(:exec)
+      .with("ssh -t root@1.1.1.1 'docker exec -it app-web-999 ruby -v'")
+    run_command("exec", "-i", "--reuse", "ruby -v").tap do |output|
+      assert_match "Get current version of running container...", output
+      assert_match "Running docker ps --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --latest --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done on 1.1.1.1", output
+      assert_match "Launching interactive command with version 999 via SSH from existing container on 1.1.1.1...", output
+    end
+  end
+
   test "containers" do
     run_command("containers").tap do |output|
       assert_match "docker container ls --all --filter label=service=app", output

test/cli/build_test.rb

@@ -57,6 +57,7 @@ class CliBuildTest < CliTestCase
     run_command("pull").tap do |output|
       assert_match /docker image rm --force dhh\/app:999/, output
       assert_match /docker pull dhh\/app:999/, output
+      assert_match "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:999 | grep -x app || (echo \"Image dhh/app:999 is missing the `service` label\" && exit 1)", output
     end
   end
 
@@ -66,6 +67,14 @@ class CliBuildTest < CliTestCase
     end
   end
 
+  test "create remote" do
+    run_command("create", fixture: :with_remote_builder).tap do |output|
+      assert_match "Running /usr/bin/env true on 1.1.1.5", output
+      assert_match "docker context create kamal-app-native-remote-amd64 --description 'kamal-app-native-remote amd64 native host' --docker 'host=ssh://app@1.1.1.5'", output
+      assert_match "docker buildx create --name kamal-app-native-remote kamal-app-native-remote-amd64 --platform linux/amd64", output
+    end
+  end
+
   test "create with error" do
     stub_setup
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
@@ -95,8 +104,8 @@ class CliBuildTest < CliTestCase
   end
 
   private
-    def run_command(*command)
+    def run_command(*command, fixture: :with_accessories)
-      stdouted { Kamal::Cli::Build.start([*command, "-c", "test/fixtures/deploy_with_accessories.yml"]) }
+      stdouted { Kamal::Cli::Build.start([*command, "-c", "test/fixtures/deploy_#{fixture}.yml"]) }
     end
 
     def stub_dependency_checks

test/cli/healthcheck_test.rb

@@ -5,7 +5,7 @@ class CliHealthcheckTest < CliTestCase
     # Prevent expected failures from outputting to terminal
     Thread.report_on_exception = false
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
     Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012")
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
@@ -35,7 +35,7 @@ class CliHealthcheckTest < CliTestCase
     # Prevent expected failures from outputting to terminal
     Thread.report_on_exception = false
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep) # No sleeping when retrying
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep) # No sleeping when retrying
 
     SSHKit::Backend::Abstract.any_instance.stubs(:execute)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^healthcheck-app-999$", "--quiet", "|", :xargs, :docker, :stop, raise_on_non_zero_exit: false)

test/cli/main_test.rb

@@ -3,6 +3,7 @@ require_relative "cli_test_case"
 class CliMainTest < CliTestCase
   test "setup" do
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap")
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push")
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ])
     Kamal::Cli::Main.any_instance.expects(:deploy)
 
@@ -16,7 +17,7 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
@@ -43,7 +44,7 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
@@ -113,7 +114,7 @@ class CliMainTest < CliTestCase
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
 
@@ -123,17 +124,25 @@ class CliMainTest < CliTestCase
   end
 
   test "deploy with missing secrets" do
-    assert_raises(KeyError) do
+    invoke_options = { "config_file" => "test/fixtures/deploy_with_secrets.yml", "version" => "999", "skip_hooks" => false }
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options)
+
     run_command("deploy", config_file: "deploy_with_secrets")
   end
-  end
 
   test "redeploy" do
     invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false }
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
 
     Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true)
@@ -155,7 +164,7 @@ class CliMainTest < CliTestCase
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:pull", [], invoke_options)
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options)
-    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options)
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true))
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options)
 
     run_command("redeploy", "--skip_push").tap do |output|
@@ -193,7 +202,7 @@ class CliMainTest < CliTestCase
     end
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
-      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
       .returns("corddirectory").at_least_once # health check
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
@@ -215,7 +224,7 @@ class CliMainTest < CliTestCase
   test "rollback without old version" do
     Kamal::Cli::Main.any_instance.stubs(:container_available?).returns(true)
 
-    Kamal::Utils::HealthcheckPoller.stubs(:sleep)
+    Kamal::Cli::Healthcheck::Poller.stubs(:sleep)
 
     SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
       .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", raise_on_non_zero_exit: false)
@@ -345,6 +354,20 @@ class CliMainTest < CliTestCase
     run_command("envify")
   end
 
+  test "envify with blank line trimming" do
+    file = <<~EOF
+      HELLO=<%= 'world' %>
+      <% if true -%>
+      KEY=value
+      <% end  -%>
+    EOF
+
+    File.expects(:read).with(".env.erb").returns(file.strip)
+    File.expects(:write).with(".env", "HELLO=world\nKEY=value\n", perm: 0600)
+
+    run_command("envify")
+  end
+
   test "envify with destination" do
     File.expects(:read).with(".env.world.erb").returns("HELLO=<%= 'world' %>")
     File.expects(:write).with(".env.world", "HELLO=world", perm: 0600)
@@ -352,6 +375,14 @@ class CliMainTest < CliTestCase
     run_command("envify", "-d", "world", config_file: "deploy_for_dest")
   end
 
+  test "envify with skip_push" do
+    File.expects(:read).with(".env.erb").returns("HELLO=<%= 'world' %>")
+    File.expects(:write).with(".env", "HELLO=world", perm: 0600)
+
+    Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push").never
+    run_command("envify", "--skip-push")
+  end
+
   test "remove with confirmation" do
     run_command("remove", "-y", config_file: "deploy_with_accessories").tap do |output|
       assert_match /docker container stop traefik/, output

test/cli/prune_test.rb

@@ -10,7 +10,7 @@ class CliPruneTest < CliTestCase
 
   test "images" do
     run_command("images").tap do |output|
-      assert_match "docker image prune --force --filter label=service=app --filter dangling=true on 1.1.1.", output
+      assert_match "docker image prune --force --filter label=service=app on 1.1.1.", output
       assert_match "docker image ls --filter label=service=app --format '{{.ID}} {{.Repository}}:{{.Tag}}' | grep -v -w \"$(docker container ls -a --format '{{.Image}}\\|' --filter label=service=app | tr -d '\\n')dhh/app:latest\\|dhh/app:<none>\" | while read image tag; do docker rmi $tag; done on 1.1.1.", output
     end
   end
@@ -18,6 +18,7 @@ class CliPruneTest < CliTestCase
   test "containers" do
     run_command("containers").tap do |output|
       assert_match /docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done on 1.1.1.\d/, output
+      assert_match /docker container prune --force --filter label=service=healthcheck-app on 1.1.1.\d/, output
      end
   end
 

test/cli/server_test.rb

@@ -10,7 +10,7 @@ class CliServerTest < CliTestCase
 
   test "bootstrap install as non-root user" do
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ]', raise_on_non_zero_exit: false).returns(false).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', raise_on_non_zero_exit: false).returns(false).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
 
     assert_raise RuntimeError, "Docker is not installed on 1.1.1.1, 1.1.1.3, 1.1.1.4, 1.1.1.2 and can't be automatically installed without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/" do
@@ -20,7 +20,7 @@ class CliServerTest < CliTestCase
 
   test "bootstrap install as root user" do
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:docker, "-v", raise_on_non_zero_exit: false).returns(false).at_least_once
-    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ]', raise_on_non_zero_exit: false).returns(true).at_least_once
+    SSHKit::Backend::Abstract.any_instance.expects(:execute).with('[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', raise_on_non_zero_exit: false).returns(true).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:curl, "-fsSL", "https://get.docker.com", "|", :sh).at_least_once
     SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:mkdir, "-p", ".kamal").returns("").at_least_once
 

test/commander_test.rb

@@ -49,6 +49,12 @@ class CommanderTest < ActiveSupport::TestCase
     assert_equal "1.1.1.3", @kamal.primary_host
   end
 
+  test "primary_role" do
+    assert_equal "web", @kamal.primary_role
+    @kamal.specific_roles = "workers"
+    assert_equal "workers", @kamal.primary_role
+  end
+
   test "roles_on" do
     assert_equal [ "web" ], @kamal.roles_on("1.1.1.1")
     assert_equal [ "workers" ], @kamal.roles_on("1.1.1.3")

test/commands/app_test.rb

@@ -14,13 +14,13 @@ class CommandsAppTest < ActiveSupport::TestCase
 
   test "run" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with hostname" do
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 --hostname myhost -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run(hostname: "myhost").join(" ")
   end
 
@@ -28,7 +28,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:volumes] = ["/local/path:/container/path" ]
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --volume /local/path:/container/path --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -36,7 +36,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "path" => "/healthz" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/healthz || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -44,7 +44,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:healthcheck] = { "cmd" => "/bin/up" }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/up) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -52,14 +52,14 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "healthcheck" => { "cmd" => "/bin/healthy" } } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(/bin/healthy) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
   test "run with custom options" do
     @config[:servers] = { "web" => [ "1.1.1.1" ], "jobs" => { "hosts" => [ "1.1.1.2" ], "cmd" => "bin/jobs", "options" => { "mount" => "somewhere", "cap-add" => true } } }
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" --env-file .kamal/env/roles/app-jobs.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
+      "docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-jobs.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
       new_command(role: "jobs").run.join(" ")
   end
 
@@ -67,7 +67,7 @@ class CommandsAppTest < ActiveSupport::TestCase
     @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } }
 
     assert_equal \
-      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
+      "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --label service=\"app\" --label role=\"web\" --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
       new_command.run.join(" ")
   end
 
@@ -317,10 +317,10 @@ class CommandsAppTest < ActiveSupport::TestCase
       new_command.remove_images.join(" ")
   end
 
-  test "tag_current_as_latest" do
+  test "tag_current_image_as_latest" do
     assert_equal \
       "docker tag dhh/app:999 dhh/app:latest",
-      new_command.tag_current_as_latest.join(" ")
+      new_command.tag_current_image_as_latest.join(" ")
   end
 
   test "make_env_directory" do
@@ -332,7 +332,7 @@ class CommandsAppTest < ActiveSupport::TestCase
   end
 
   test "cord" do
-    assert_equal "docker inspect -f '{{ range .Mounts }}{{ .Source }} {{ .Destination }} {{ end }}' app-web-123 | awk '$2 == \"/tmp/kamal-cord\" {print $1}'", new_command.cord(version: 123).join(" ")
+    assert_equal "docker inspect -f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}' app-web-123 | awk '$2 == \"/tmp/kamal-cord\" {print $1}'", new_command.cord(version: 123).join(" ")
   end
 
   test "tie cord" do
@@ -345,8 +345,39 @@ class CommandsAppTest < ActiveSupport::TestCase
     assert_equal "rm -r corddir", new_command.cut_cord("corddir").join(" ")
   end
 
+  test "extract assets" do
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/extracted/app-web-999", "&&",
+      :docker, :stop, "-t 1", "app-web-assets", "2> /dev/null", "|| true", "&&",
+      :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "dhh/app:latest", "sleep 1000000", "&&",
+      :docker, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/assets/extracted/app-web-999", "&&",
+      :docker, :stop, "-t 1", "app-web-assets"
+    ], new_command(asset_path: "/public/assets").extract_assets
+  end
+
+  test "sync asset volumes" do
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-999", ".kamal/assets/volumes/app-web-999"
+    ], new_command(asset_path: "/public/assets").sync_asset_volumes
+
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-999", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-999", ".kamal/assets/volumes/app-web-998", "|| true", ";",
+      :cp, "-rnT", ".kamal/assets/extracted/app-web-998", ".kamal/assets/volumes/app-web-999", "|| true",
+    ], new_command(asset_path: "/public/assets").sync_asset_volumes(old_version: 998)
+  end
+
+  test "clean up assets" do
+    assert_equal [
+      :find, ".kamal/assets/extracted", "-maxdepth 1", "-name", "'app-web-*'", "!", "-name", "app-web-999", "-exec rm -rf \"{}\" +", ";",
+      :find, ".kamal/assets/volumes", "-maxdepth 1", "-name", "'app-web-*'", "!", "-name", "app-web-999", "-exec rm -rf \"{}\" +"
+    ], new_command(asset_path: "/public/assets").clean_up_assets
+  end
+
   private
-    def new_command(role: "web")
+    def new_command(role: "web", **additional_config)
-      Kamal::Commands::App.new(Kamal::Configuration.new(@config, destination: @destination, version: "999"), role: role)
+      Kamal::Commands::App.new(Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999"), role: role)
     end
 end

test/commands/builder_test.rb

@@ -103,6 +103,10 @@ class CommandsBuilderTest < ActiveSupport::TestCase
       builder.push.join(" ")
   end
 
+  test "validate image" do
+    assert_equal "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:123 | grep -x app || (echo \"Image dhh/app:123 is missing the `service` label\" && exit 1)", new_builder_command.validate_image.join(" ")
+  end
+
   private
     def new_builder_command(additional_config = {})
       Kamal::Commands::Builder.new(Kamal::Configuration.new(@config.merge(additional_config), version: "123"))

test/commands/docker_test.rb

@@ -21,6 +21,6 @@ class CommandsDockerTest < ActiveSupport::TestCase
   end
 
   test "superuser?" do
-    assert_equal '[ "${EUID:-$(id -u)}" -eq 0 ]', @docker.superuser?.join(" ")
+    assert_equal '[ "${EUID:-$(id -u)}" -eq 0 ] || command -v sudo >/dev/null || command -v su >/dev/null', @docker.superuser?.join(" ")
   end
 end

test/commands/healthcheck_test.rb

@@ -92,6 +92,13 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
       new_command.logs.join(" ")
   end
 
+  test "logs with custom lines number" do
+    @config[:healthcheck] = { "log_lines" => 150 }
+    assert_equal \
+      "docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker logs --tail 150 2>&1",
+      new_command.logs.join(" ")
+  end
+
   test "logs with destination" do
     @destination = "staging"
 

test/commands/prune_test.rb

@@ -10,7 +10,7 @@ class CommandsPruneTest < ActiveSupport::TestCase
 
   test "dangling images" do
     assert_equal \
-      "docker image prune --force --filter label=service=app --filter dangling=true",
+      "docker image prune --force --filter label=service=app",
       new_command.dangling_images.join(" ")
   end
 
@@ -20,10 +20,16 @@ class CommandsPruneTest < ActiveSupport::TestCase
       new_command.tagged_images.join(" ")
   end
 
-  test "containers" do
+  test "app containers" do
     assert_equal \
       "docker ps -q -a --filter label=service=app --filter status=created --filter status=exited --filter status=dead | tail -n +6 | while read container_id; do docker rm $container_id; done",
-      new_command.containers.join(" ")
+      new_command.app_containers.join(" ")
+  end
+
+  test "healthcheck containers" do
+    assert_equal \
+      "docker container prune --force --filter label=service=healthcheck-app",
+      new_command.healthcheck_containers.join(" ")
   end
 
   private

test/commands/traefik_test.rb

@@ -180,7 +180,7 @@ class CommandsTraefikTest < ActiveSupport::TestCase
   test "env_file" do
     @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] }
 
-    assert_equal "EXAMPLE_API_KEY=456\n", new_command.env_file
+    assert_equal "EXAMPLE_API_KEY=456\n", new_command.env_file.to_s
   end
 
   test "host_env_file_path" do

test/configuration/accessory_test.rb

@@ -123,7 +123,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase
       MYSQL_ROOT_HOST=%
     ENV
 
-    assert_equal expected, @config.accessory(:mysql).env_file
+    assert_equal expected, @config.accessory(:mysql).env_file.to_s
   ensure
     ENV["MYSQL_ROOT_PASSWORD"] = nil
   end

test/configuration/role_test.rb

@@ -77,7 +77,16 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected_env, @config_with_roles.role(:workers).env_file
+    assert_equal expected_env, @config_with_roles.role(:workers).env_file.to_s
+  end
+
+  test "container name" do
+    ENV["VERSION"] = "12345"
+
+    assert_equal "app-workers-12345", @config_with_roles.role(:workers).container_name
+    assert_equal "app-web-12345", @config_with_roles.role(:web).container_name
+  ensure
+    ENV.delete("VERSION")
   end
 
   test "env args" do
@@ -114,7 +123,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected, @config_with_roles.role(:workers).env_file
+    assert_equal expected, @config_with_roles.role(:workers).env_file.to_s
   ensure
     ENV["REDIS_PASSWORD"] = nil
     ENV["DB_PASSWORD"] = nil
@@ -139,7 +148,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected, @config_with_roles.role(:workers).env_file
+    assert_equal expected, @config_with_roles.role(:workers).env_file.to_s
   ensure
     ENV["DB_PASSWORD"] = nil
   end
@@ -162,7 +171,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
       WEB_CONCURRENCY=4
     ENV
 
-    assert_equal expected, @config_with_roles.role(:workers).env_file
+    assert_equal expected, @config_with_roles.role(:workers).env_file.to_s
   ensure
     ENV["REDIS_PASSWORD"] = nil
   end
@@ -180,19 +189,67 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     assert !@config_with_roles.role(:workers).uses_cord?
   end
 
-  test "cord host directory" do
-    assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}}, @config_with_roles.role(:web).cord_host_directory
-  end
-
   test "cord host file" do
-    assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}/cord}, @config_with_roles.role(:web).cord_host_file
+    assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}/cord}, @config_with_roles.role(:web).cord_host_file
   end
 
-  test "cord container directory" do
+  test "cord volume" do
-    assert_equal "/tmp/kamal-cord", @config_with_roles.role(:web).cord_container_directory
+    assert_equal "/tmp/kamal-cord", @config_with_roles.role(:web).cord_volume.container_path
+    assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}}, @config_with_roles.role(:web).cord_volume.host_path
+    assert_equal "--volume", @config_with_roles.role(:web).cord_volume.docker_args[0]
+    assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}:/tmp/kamal-cord}, @config_with_roles.role(:web).cord_volume.docker_args[1]
   end
 
   test "cord container file" do
     assert_equal "/tmp/kamal-cord/cord", @config_with_roles.role(:web).cord_container_file
   end
+
+  test "asset path and volume args" do
+    ENV["VERSION"] = "12345"
+    assert_nil @config_with_roles.role(:web).asset_volume_args
+    assert_nil @config_with_roles.role(:workers).asset_volume_args
+    assert_nil @config_with_roles.role(:web).asset_path
+    assert_nil @config_with_roles.role(:workers).asset_path
+    assert !@config_with_roles.role(:web).assets?
+    assert !@config_with_roles.role(:workers).assets?
+
+    config_with_assets = Kamal::Configuration.new(@deploy_with_roles.dup.tap { |c|
+      c[:asset_path] = "foo"
+    })
+    assert_equal "foo", config_with_assets.role(:web).asset_path
+    assert_equal "foo", config_with_assets.role(:workers).asset_path
+    assert_equal ["--volume", "$(pwd)/.kamal/assets/volumes/app-web-12345:foo"], config_with_assets.role(:web).asset_volume_args
+    assert_nil config_with_assets.role(:workers).asset_volume_args
+    assert config_with_assets.role(:web).assets?
+    assert !config_with_assets.role(:workers).assets?
+
+    config_with_assets = Kamal::Configuration.new(@deploy_with_roles.dup.tap { |c|
+      c[:servers]["web"] = { "hosts" => [ "1.1.1.1", "1.1.1.2" ], "asset_path" => "bar" }
+    })
+    assert_equal "bar", config_with_assets.role(:web).asset_path
+    assert_nil config_with_assets.role(:workers).asset_path
+    assert_equal ["--volume", "$(pwd)/.kamal/assets/volumes/app-web-12345:bar"], config_with_assets.role(:web).asset_volume_args
+    assert_nil config_with_assets.role(:workers).asset_volume_args
+    assert config_with_assets.role(:web).assets?
+    assert !config_with_assets.role(:workers).assets?
+
+  ensure
+    ENV.delete("VERSION")
+  end
+
+  test "asset extracted path" do
+    ENV["VERSION"] = "12345"
+    assert_equal ".kamal/assets/extracted/app-web-12345", @config_with_roles.role(:web).asset_extracted_path
+    assert_equal ".kamal/assets/extracted/app-workers-12345", @config_with_roles.role(:workers).asset_extracted_path
+  ensure
+    ENV.delete("VERSION")
+  end
+
+  test "asset volume path" do
+    ENV["VERSION"] = "12345"
+    assert_equal ".kamal/assets/volumes/app-web-12345", @config_with_roles.role(:web).asset_volume_path
+    assert_equal ".kamal/assets/volumes/app-workers-12345", @config_with_roles.role(:workers).asset_volume_path
+  ensure
+    ENV.delete("VERSION")
+  end
 end

test/configuration/volume_test.rb

@@ -0,0 +1,13 @@
+require "test_helper"
+
+class ConfigurationVolumeTest < ActiveSupport::TestCase
+  test "docker args absolute" do
+    volume = Kamal::Configuration::Volume.new(host_path: "/root/foo/bar", container_path: "/assets")
+    assert_equal ["--volume", "/root/foo/bar:/assets"], volume.docker_args
+  end
+
+  test "docker args relative" do
+    volume = Kamal::Configuration::Volume.new(host_path: "foo/bar", container_path: "/assets")
+    assert_equal ["--volume", "$(pwd)/foo/bar:/assets"], volume.docker_args
+  end
+end

test/configuration_test.rb

@@ -75,7 +75,7 @@ class ConfigurationTest < ActiveSupport::TestCase
   test "version no git repo" do
     ENV.delete("VERSION")
 
-    @config.expects(:system).with("git rev-parse").returns(nil)
+    Kamal::Git.expects(:used?).returns(nil)
     error = assert_raises(RuntimeError) { @config.version}
     assert_match /no git repository found/, error.message
   end
@@ -83,16 +83,16 @@ class ConfigurationTest < ActiveSupport::TestCase
   test "version from git committed" do
     ENV.delete("VERSION")
 
-    @config.expects(:`).with("git rev-parse HEAD").returns("git-version")
+    Kamal::Git.expects(:revision).returns("git-version")
-    Kamal::Utils.expects(:uncommitted_changes).returns("")
+    Kamal::Git.expects(:uncommitted_changes).returns("")
     assert_equal "git-version", @config.version
   end
 
   test "version from git uncommitted" do
     ENV.delete("VERSION")
 
-    @config.expects(:`).with("git rev-parse HEAD").returns("git-version")
+    Kamal::Git.expects(:revision).returns("git-version")
-    Kamal::Utils.expects(:uncommitted_changes).returns("M   file\n")
+    Kamal::Git.expects(:uncommitted_changes).returns("M   file\n")
     assert_match /^git-version_uncommitted_[0-9a-f]{16}$/, @config.version
   end
 
@@ -124,12 +124,8 @@ class ConfigurationTest < ActiveSupport::TestCase
     assert_equal "app-missing", @config.service_with_version
   end
 
-  test "env with missing secret" do
+  test "healthcheck service" do
-    assert_raises(KeyError) do
+    assert_equal "healthcheck-app", @config.healthcheck_service
-      config = Kamal::Configuration.new(@deploy.tap { |c| c.merge!({
-        env: { "secret" => [ "PASSWORD" ] }
-      }) }).ensure_env_available
-    end
   end
 
   test "valid config" do
@@ -210,6 +206,18 @@ class ConfigurationTest < ActiveSupport::TestCase
     end
   end
 
+  test "destination required" do
+    dest_config_file = Pathname.new(File.expand_path("fixtures/deploy_for_required_dest.yml", __dir__))
+
+    assert_raises(ArgumentError) do
+      config = Kamal::Configuration.create_from config_file: dest_config_file
+    end
+
+    assert_nothing_raised do
+      config = Kamal::Configuration.create_from config_file: dest_config_file, destination: "world"
+    end
+  end
+
   test "to_h" do
     expected_config = \
       { :roles=>["web"],
@@ -219,12 +227,12 @@ class ConfigurationTest < ActiveSupport::TestCase
         :repository=>"dhh/app",
         :absolute_image=>"dhh/app:missing",
         :service_with_version=>"app-missing",
-        :ssh_options=>{ :user=>"root", :auth_methods=>["publickey"], log_level: :fatal, keepalive: true, keepalive_interval: 30 },
+        :ssh_options=>{ :user=>"root", log_level: :fatal, keepalive: true, keepalive_interval: 30 },
         :sshkit=>{},
         :volume_args=>["--volume", "/local/path:/container/path"],
         :builder=>{},
         :logging=>["--log-opt", "max-size=\"10m\""],
-        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord" }}
+        :healthcheck=>{ "path"=>"/up", "port"=>3000, "max_attempts" => 7, "exposed_port" => 3999, "cord" => "/tmp/kamal-cord", "log_lines" => 50 }}
 
     assert_equal expected_config, @config.to_h
   end
@@ -265,4 +273,9 @@ class ConfigurationTest < ActiveSupport::TestCase
     SecureRandom.expects(:hex).with(16).returns("09876543211234567890098765432112")
     assert_equal "09876543211234567890098765432112", @config.run_id
   end
+
+  test "asset path" do
+    assert_nil @config.asset_path
+    assert_equal "foo", Kamal::Configuration.new(@deploy.merge!(asset_path: "foo")).asset_path
+  end
 end

test/env_file_test.rb

@@ -0,0 +1,102 @@
+require "test_helper"
+
+class EnvFileTest < ActiveSupport::TestCase
+  test "env file simple" do
+    env = {
+      "foo" => "bar",
+      "baz" => "haz"
+    }
+
+    assert_equal "foo=bar\nbaz=haz\n", \
+      Kamal::EnvFile.new(env).to_s
+  end
+
+  test "env file clear" do
+    env = {
+      "clear" => {
+        "foo" => "bar",
+        "baz" => "haz"
+      }
+    }
+
+    assert_equal "foo=bar\nbaz=haz\n", \
+      Kamal::EnvFile.new(env).to_s
+  end
+
+  test "env file empty" do
+    assert_equal "\n", Kamal::EnvFile.new({}).to_s
+  end
+
+  test "env file secret" do
+    ENV["PASSWORD"] = "hello"
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_equal "PASSWORD=hello\n", \
+      Kamal::EnvFile.new(env).to_s
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "env file secret escaped newline" do
+    ENV["PASSWORD"] = "hello\\nthere"
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_equal "PASSWORD=hello\\\\nthere\n", \
+      Kamal::EnvFile.new(env).to_s
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "env file secret newline" do
+    ENV["PASSWORD"] = "hello\nthere"
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_equal "PASSWORD=hello\\nthere\n", \
+      Kamal::EnvFile.new(env).to_s
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "env file missing secret" do
+    env = {
+      "secret" => [ "PASSWORD" ]
+    }
+
+    assert_raises(KeyError) { Kamal::EnvFile.new(env).to_s }
+
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "env file secret and clear" do
+    ENV["PASSWORD"] = "hello"
+    env = {
+      "secret" => [ "PASSWORD" ],
+      "clear" => {
+        "foo" => "bar",
+        "baz" => "haz"
+      }
+    }
+
+    assert_equal "PASSWORD=hello\nfoo=bar\nbaz=haz\n", \
+      Kamal::EnvFile.new(env).to_s
+  ensure
+    ENV.delete "PASSWORD"
+  end
+
+  test "stringIO conversion" do
+    env = {
+      "foo" => "bar",
+      "baz" => "haz"
+    }
+
+    assert_equal "foo=bar\nbaz=haz\n", \
+      StringIO.new(Kamal::EnvFile.new(env)).read
+  end
+end

test/fixtures/deploy_for_required_dest.world.yml

@@ -0,0 +1,5 @@
+servers:
+  - 1.1.1.1
+  - 1.1.1.2
+env:
+  REDIS_URL: redis://x/y

test/fixtures/deploy_for_required_dest.yml

@@ -0,0 +1,7 @@
+service: app
+image: dhh/app
+registry:
+  server: registry.digitalocean.com
+  username: <%= "my-user" %>
+  password: <%= "my-password" %>
+require_destination: true

test/fixtures/deploy_with_assets.yml

@@ -0,0 +1,9 @@
+service: app
+image: dhh/app
+servers:
+  - "1.1.1.1"
+  - "1.1.1.2"
+registry:
+  username: user
+  password: pw
+asset_path: /public/assets

test/fixtures/deploy_with_remote_builder.yml

@@ -0,0 +1,41 @@
+service: app
+image: dhh/app
+servers:
+  web:
+    - "1.1.1.1"
+    - "1.1.1.2"
+  workers:
+    - "1.1.1.3"
+    - "1.1.1.4"
+registry:
+  username: user
+  password: pw
+
+accessories:
+  mysql:
+    image: mysql:5.7
+    host: 1.1.1.3
+    port: 3306
+    env:
+      clear:
+        MYSQL_ROOT_HOST: '%'
+      secret:
+        - MYSQL_ROOT_PASSWORD
+    files:
+      - test/fixtures/files/my.cnf:/etc/mysql/my.cnf
+    directories:
+      - data:/var/lib/mysql
+  redis:
+    image: redis:latest
+    roles:
+      - web
+    port: 6379
+    directories:
+      - data:/data
+
+readiness_delay: 0
+
+builder:
+  remote:
+    arch: amd64
+    host: ssh://app@1.1.1.5

test/git_test.rb

@@ -0,0 +1,13 @@
+require "test_helper"
+
+class GitTest < ActiveSupport::TestCase
+  test "uncommitted changes exist" do
+    Kamal::Git.expects(:`).with("git status --porcelain").returns("M   file\n")
+    assert_equal "M   file", Kamal::Git.uncommitted_changes
+  end
+
+  test "uncommitted changes do not exist" do
+    Kamal::Git.expects(:`).with("git status --porcelain").returns("")
+    assert_equal "", Kamal::Git.uncommitted_changes
+  end
+end

test/integration/docker/deployer/app/Dockerfile

@@ -4,4 +4,6 @@ COPY default.conf /etc/nginx/conf.d/default.conf
 
 ARG COMMIT_SHA
 RUN echo $COMMIT_SHA > /usr/share/nginx/html/version
+RUN mkdir -p /usr/share/nginx/html/versions && echo "version" > /usr/share/nginx/html/versions/$COMMIT_SHA
+RUN mkdir -p /usr/share/nginx/html/versions && echo "hidden" > /usr/share/nginx/html/versions/.hidden
 

test/integration/docker/deployer/app/config/deploy.yml

@@ -8,6 +8,7 @@ env:
     CLEAR_TOKEN: '4321'
   secret:
     - SECRET_TOKEN
+asset_path: /usr/share/nginx/html/versions
 
 registry:
   server: registry:4443

test/integration/docker/shared/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.10
+FROM ubuntu:22.04
 
 WORKDIR /work
 

test/integration/docker/vm/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.10
+FROM ubuntu:22.04
 
 WORKDIR /work
 

test/integration/main_test.rb

@@ -5,6 +5,7 @@ class MainTest < IntegrationTest
     kamal :envify
     assert_local_env_file "SECRET_TOKEN=1234"
     assert_remote_env_file "SECRET_TOKEN=1234\nCLEAR_TOKEN=4321"
+    remove_local_env_file
 
     first_version = latest_app_version
 
@@ -20,6 +21,8 @@ class MainTest < IntegrationTest
     assert_app_is_up version: second_version
     assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy"
 
+    assert_accumulated_assets first_version, second_version
+
     kamal :rollback, first_version
     assert_hooks_ran "pre-connect", "pre-deploy", "post-deploy"
     assert_app_is_up version: first_version
@@ -51,10 +54,10 @@ class MainTest < IntegrationTest
     assert_equal "registry:4443/app:#{version}", config[:absolute_image]
     assert_equal "app-#{version}", config[:service_with_version]
     assert_equal [], config[:volume_args]
-    assert_equal({ user: "root", auth_methods: [ "publickey" ], keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])
+    assert_equal({ user: "root", keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options])
     assert_equal({ "multiarch" => false, "args" => { "COMMIT_SHA" => version } }, config[:builder])
     assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging]
-    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord"=>"/tmp/kamal-cord", "cmd"=>"wget -qO- http://localhost > /dev/null || exit 1" }, config[:healthcheck])
+    assert_equal({ "path" => "/up", "port" => 3000, "max_attempts" => 7, "exposed_port" => 3999, "cord"=>"/tmp/kamal-cord", "log_lines" => 50, "cmd"=>"wget -qO- http://localhost > /dev/null || exit 1" }, config[:healthcheck])
   end
 
   private
@@ -62,6 +65,10 @@ class MainTest < IntegrationTest
       assert_equal contents, deployer_exec("cat .env", capture: true)
     end
 
+    def remove_local_env_file
+      deployer_exec("rm .env")
+    end
+
     def assert_remote_env_file(contents)
       assert_equal contents, docker_compose("exec vm1 cat /root/.kamal/env/roles/app-web.env", capture: true)
     end
@@ -69,4 +76,12 @@ class MainTest < IntegrationTest
     def assert_no_remote_env_file
       assert_equal "nofile", docker_compose("exec vm1 stat /root/.kamal/env/roles/app-web.env 2> /dev/null || echo nofile", capture: true)
     end
+
+    def assert_accumulated_assets(*versions)
+      versions.each do |version|
+        assert_equal "200", Net::HTTP.get_response(URI.parse("http://localhost:12345/versions/#{version}")).code
+      end
+
+      assert_equal "200", Net::HTTP.get_response(URI.parse("http://localhost:12345/versions/.hidden")).code
+    end
 end

test/utils_test.rb

@@ -11,67 +11,6 @@ class UtilsTest < ActiveSupport::TestCase
       Kamal::Utils.argumentize("--label", { foo: "bar" }, sensitive: true).last
   end
 
-  test "env file simple" do
-    env = {
-      "foo" => "bar",
-      "baz" => "haz"
-    }
-
-    assert_equal "foo=bar\nbaz=haz\n", \
-      Kamal::Utils.env_file_with_secrets(env)
-  end
-
-  test "env file clear" do
-    env = {
-      "clear" => {
-        "foo" => "bar",
-        "baz" => "haz"
-      }
-    }
-
-    assert_equal "foo=bar\nbaz=haz\n", \
-      Kamal::Utils.env_file_with_secrets(env)
-  end
-
-  test "env file secret" do
-    ENV["PASSWORD"] = "hello"
-    env = {
-      "secret" => [ "PASSWORD" ]
-    }
-
-    assert_equal "PASSWORD=hello\n", \
-      Kamal::Utils.env_file_with_secrets(env)
-  ensure
-    ENV.delete "PASSWORD"
-  end
-
-  test "env file missing secret" do
-    env = {
-      "secret" => [ "PASSWORD" ]
-    }
-
-    assert_raises(KeyError) { Kamal::Utils.env_file_with_secrets(env) }
-
-  ensure
-    ENV.delete "PASSWORD"
-  end
-
-  test "env file secret and clear" do
-    ENV["PASSWORD"] = "hello"
-    env = {
-      "secret" => [ "PASSWORD" ],
-      "clear" => {
-        "foo" => "bar",
-        "baz" => "haz"
-      }
-    }
-
-    assert_equal "PASSWORD=hello\nfoo=bar\nbaz=haz\n", \
-      Kamal::Utils.env_file_with_secrets(env)
-  ensure
-    ENV.delete "PASSWORD"
-  end
-
   test "optionize" do
     assert_equal [ "--foo", "\"bar\"", "--baz", "\"qux\"", "--quux" ], \
       Kamal::Utils.optionize({ foo: "bar", baz: "qux", quux: true })
@@ -113,14 +52,4 @@ class UtilsTest < ActiveSupport::TestCase
     assert_equal "\"https://example.com/\\$2\"",
       Kamal::Utils.escape_shell_value("https://example.com/$2")
   end
-
-  test "uncommitted changes exist" do
-    Kamal::Utils.expects(:`).with("git status --porcelain").returns("M   file\n")
-    assert_equal "M   file", Kamal::Utils.uncommitted_changes
-  end
-
-  test "uncommitted changes do not exist" do
-    Kamal::Utils.expects(:`).with("git status --porcelain").returns("")
-    assert_equal "", Kamal::Utils.uncommitted_changes
-  end
 end