From 0b439362da6f1822fe4aefe8f7f1fc05dcc8b538 Mon Sep 17 00:00:00 2001
From: Donal McBreen <donal@basecamp.com>
Date: Thu, 7 Sep 2023 15:07:43 +0100
Subject: [PATCH] Asset paths

During deployments both the old and new containers will be active for a
small period of time. There also may be lagging requests for older CSS
and JS after the deployment.

This can lead to 404s if a request for old assets hits a new container
or visa-versa.

This PR makes sure that both sets of assets are available throughout the
deployment from before the new version of the app is booted.

This can be configured by setting the asset path:

```yaml
asset_path: "/rails/public/assets"
```

The process is:
1. We extract the assets out of the container, with docker run, docker
cp, docker stop. Docker run sets the container command to "sleep" so
this needs to be available in the container.
2. We create an asset volume directory on the host for the new version
of the app on the host and copy the assets in there.
3. If there is a previous deployment we also copy the new assets into
its asset volume and copy the older assets into the new asset volume.
4. We start the new container mapping the asset volume over the top of
the container's asset path.

This means the both the old and new versions have replaced the asset
path with a volume containing both sets of assets and should be able
to serve any request during the deployment. The older assets will
continue to be available until the next deployment.
---
 lib/kamal/cli/app.rb                          | 26 ++++---
 lib/kamal/commands/app.rb                     | 56 ++++++++++++++-
 lib/kamal/configuration.rb                    |  4 ++
 lib/kamal/configuration/role.rb               | 51 +++++++++++--
 lib/kamal/configuration/volume.rb             | 22 ++++++
 test/cli/app_test.rb                          | 30 +++++++-
 test/commands/app_test.rb                     | 35 ++++++++-
 test/configuration/role_test.rb               | 71 +++++++++++++++++--
 test/configuration/volume_test.rb             | 13 ++++
 test/configuration_test.rb                    |  5 ++
 test/fixtures/deploy_with_assets.yml          |  9 +++
 .../docker/deployer/app/Dockerfile            |  1 +
 .../docker/deployer/app/config/deploy.yml     |  1 +
 test/integration/main_test.rb                 |  8 +++
 14 files changed, 304 insertions(+), 28 deletions(-)
 create mode 100644 lib/kamal/configuration/volume.rb
 create mode 100644 test/configuration/volume_test.rb
 create mode 100644 test/fixtures/deploy_with_assets.yml

diff --git a/lib/kamal/cli/app.rb b/lib/kamal/cli/app.rb
index 9c34e8dd..3d043d72 100644
--- a/lib/kamal/cli/app.rb
+++ b/lib/kamal/cli/app.rb
@@ -10,12 +10,21 @@ class Kamal::Cli::App < Kamal::Cli::Base
           on(KAMAL.hosts) do
             execute *KAMAL.auditor.record("Tagging #{KAMAL.config.absolute_image} as the latest image"), verbosity: :debug
             execute *KAMAL.app.tag_current_as_latest
+
+            KAMAL.roles_on(host).each do |role|
+              app = KAMAL.app(role: role)
+              role_config = KAMAL.config.role(role)
+
+              if role_config.assets?
+                execute *app.extract_assets
+                old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
+                execute *app.sync_asset_volumes(old_version: old_version)
+              end
+            end
           end
 
           on(KAMAL.hosts, **KAMAL.boot_strategy) do |host|
-            roles = KAMAL.roles_on(host)
-
-            roles.each do |role|
+            KAMAL.roles_on(host).each do |role|
               app = KAMAL.app(role: role)
               auditor = KAMAL.auditor(role: role)
               role_config = KAMAL.config.role(role)
@@ -27,13 +36,11 @@ class Kamal::Cli::App < Kamal::Cli::Base
                 execute *app.rename_container(version: version, new_version: tmp_version)
               end
 
-              execute *auditor.record("Booted app version #{version}"), verbosity: :debug
-
               old_version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip
 
-              if role_config.uses_cord?
-                execute *app.tie_cord(role_config.cord_host_file)
-              end
+              execute *app.tie_cord(role_config.cord_host_file) if role_config.uses_cord?
+
+              execute *auditor.record("Booted app version #{version}"), verbosity: :debug
 
               execute *app.run(hostname: "#{host}-#{SecureRandom.hex(6)}")
 
@@ -47,7 +54,10 @@ class Kamal::Cli::App < Kamal::Cli::Base
                     Kamal::Utils::HealthcheckPoller.wait_for_unhealthy(pause_after_ready: true) { capture_with_info(*app.status(version: old_version)) }
                   end
                 end
+
                 execute *app.stop(version: old_version), raise_on_non_zero_exit: false
+
+                execute *app.clean_up_assets if role_config.assets?
               end
             end
           end
diff --git a/lib/kamal/commands/app.rb b/lib/kamal/commands/app.rb
index 3a8b0789..0b79aa7b 100644
--- a/lib/kamal/commands/app.rb
+++ b/lib/kamal/commands/app.rb
@@ -20,6 +20,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
       *role_config.health_check_args,
       *config.logging_args,
       *config.volume_args,
+      *role_config.asset_volume_args,
       *role_config.label_args,
       *role_config.option_args,
       config.absolute_image,
@@ -105,7 +106,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
   def list_versions(*docker_args, statuses: nil)
     pipe \
       docker(:ps, *filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'),
-      %(while read line; do echo ${line##{role_config.full_name}-}; done) # Extract SHA from "service-role-dest-SHA"
+      %(while read line; do echo ${line##{role_config.container_prefix}-}; done) # Extract SHA from "service-role-dest-SHA"
   end
 
   def list_containers
@@ -153,7 +154,7 @@ class Kamal::Commands::App < Kamal::Commands::Base
   def cord(version:)
     pipe \
       docker(:inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\n\" .Source .Destination}}{{ end }}'", container_name(version)),
-      [:awk, "'$2 == \"#{role_config.cord_container_directory}\" {print $1}'"]
+      [:awk, "'$2 == \"#{role_config.cord_volume.container_path}\" {print $1}'"]
   end
 
   def tie_cord(cord)
@@ -164,9 +165,43 @@ class Kamal::Commands::App < Kamal::Commands::Base
     remove_directory(cord)
   end
 
+  def extract_assets
+    asset_container = "#{role_config.container_prefix}-assets"
+
+    combine \
+      make_directory(role_config.asset_extracted_path),
+      [*docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true"],
+      docker(:run, "--name", asset_container, "--detach", "--rm", config.latest_image, "sleep 1000000"),
+      docker(:cp, "-L", "#{asset_container}:#{role_config.asset_path}/.", role_config.asset_extracted_path),
+      docker(:stop, "-t 1", asset_container),
+      by: "&&"
+  end
+
+  def sync_asset_volumes(old_version: nil)
+    new_extracted_path, new_volume_path = role_config.asset_extracted_path(config.version), role_config.asset_volume.host_path
+    if old_version.present?
+      old_extracted_path, old_volume_path = role_config.asset_extracted_path(old_version), role_config.asset_volume(old_version).host_path
+    end
+
+    commands = [make_directory(new_volume_path), copy_contents(new_extracted_path, new_volume_path)]
+
+    if old_version.present?
+      commands << copy_contents(new_extracted_path, old_volume_path, continue_on_error: true)
+      commands << copy_contents(old_extracted_path, new_volume_path, continue_on_error: true)
+    end
+
+    chain *commands
+  end
+
+  def clean_up_assets
+    chain \
+      find_and_remove_older_siblings(role_config.asset_extracted_path),
+      find_and_remove_older_siblings(role_config.asset_volume_path)
+  end
+
   private
     def container_name(version = nil)
-      [ role_config.full_name, version || config.version ].compact.join("-")
+      [ role_config.container_prefix, version || config.version ].compact.join("-")
     end
 
     def filter_args(statuses: nil)
@@ -186,4 +221,19 @@ class Kamal::Commands::App < Kamal::Commands::Base
         end
       end
     end
+
+    def find_and_remove_older_siblings(path)
+      [
+        :find,
+        Pathname.new(path).dirname.to_s,
+        "-maxdepth 1",
+        "-name", "'#{role_config.container_prefix}-*'",
+        "!", "-name", Pathname.new(path).basename.to_s,
+        "-exec rm -rf \"{}\" +"
+      ]
+    end
+
+    def copy_contents(source, destination, continue_on_error: false)
+      [ :cp, "-rn", "#{source}/*", destination, *("|| true" if continue_on_error)]
+    end
 end
diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb
index af6e46c5..9c8cc4af 100644
--- a/lib/kamal/configuration.rb
+++ b/lib/kamal/configuration.rb
@@ -211,6 +211,10 @@ class Kamal::Configuration
     @run_id ||= SecureRandom.hex(16)
   end
 
+  def asset_path
+    raw_config.asset_path
+  end
+
   private
     # Will raise ArgumentError if any required config keys are missing
     def ensure_required_keys_present
diff --git a/lib/kamal/configuration/role.rb b/lib/kamal/configuration/role.rb
index 6c57bc1d..d135acec 100644
--- a/lib/kamal/configuration/role.rb
+++ b/lib/kamal/configuration/role.rb
@@ -48,11 +48,15 @@ class Kamal::Configuration::Role
     argumentize "--env-file", host_env_file_path
   end
 
+  def asset_volume_args
+    asset_volume&.docker_args
+  end
+
   def health_check_args(cord: true)
     if health_check_cmd.present?
       if cord && uses_cord?
         optionize({ "health-cmd" => health_check_cmd_with_cord, "health-interval" => health_check_interval })
-          .concat(["--volume", "#{cord_host_directory}:#{cord_container_directory}"])
+          .concat(cord_volume.docker_args)
       else
         optionize({ "health-cmd" => health_check_cmd, "health-interval" => health_check_interval })
       end
@@ -74,15 +78,23 @@ class Kamal::Configuration::Role
   end
 
   def uses_cord?
-    running_traefik? && cord_container_directory.present? && health_check_cmd.present?
+    running_traefik? && cord_volume && health_check_cmd.present?
   end
 
   def cord_host_directory
-    File.join config.run_directory_as_docker_volume, "cords", [full_name, config.run_id].join("-")
+    File.join config.run_directory_as_docker_volume, "cords", [container_prefix, config.run_id].join("-")
+  end
+
+  def cord_volume
+    if (cord = health_check_options["cord"])
+      @cord_volume ||= Kamal::Configuration::Volume.new \
+        host_path: File.join(config.run_directory, "cords", [container_prefix, config.run_id].join("-")),
+        container_path: cord
+    end
   end
 
   def cord_host_file
-    File.join cord_host_directory, CORD_FILE
+    File.join cord_volume.host_path, CORD_FILE
   end
 
   def cord_container_directory
@@ -90,7 +102,7 @@ class Kamal::Configuration::Role
   end
 
   def cord_container_file
-    File.join cord_container_directory, CORD_FILE
+    File.join cord_volume.container_path, CORD_FILE
   end
 
 
@@ -110,10 +122,37 @@ class Kamal::Configuration::Role
     name.web? || specializations["traefik"]
   end
 
-  def full_name
+  def container_name(version = nil)
+    [ container_prefix, version || config.version ].compact.join("-")
+  end
+
+  def container_prefix
     [ config.service, name, config.destination ].compact.join("-")
   end
 
+  def asset_path
+    specializations["asset_path"] || config.asset_path
+  end
+
+  def assets?
+    asset_path.present? && running_traefik?
+  end
+
+  def asset_volume(version = nil)
+    if assets?
+      Kamal::Configuration::Volume.new \
+        host_path: asset_volume_path(version), container_path: asset_path
+    end
+  end
+
+  def asset_extracted_path(version = nil)
+    File.join config.run_directory, "assets", "extracted", container_name(version)
+  end
+
+  def asset_volume_path(version = nil)
+    File.join config.run_directory, "assets", "volumes", container_name(version)
+  end
+
   private
     attr_accessor :config
 
diff --git a/lib/kamal/configuration/volume.rb b/lib/kamal/configuration/volume.rb
new file mode 100644
index 00000000..98d9398d
--- /dev/null
+++ b/lib/kamal/configuration/volume.rb
@@ -0,0 +1,22 @@
+class Kamal::Configuration::Volume
+  attr_reader :host_path, :container_path
+  delegate :argumentize, to: Kamal::Utils
+
+  def initialize(host_path:, container_path:)
+    @host_path = host_path
+    @container_path = container_path
+  end
+
+  def docker_args
+    argumentize "--volume", "#{host_path_for_docker_volume}:#{container_path}"
+  end
+
+  private
+    def host_path_for_docker_volume
+      if Pathname.new(host_path).absolute?
+        host_path
+      else
+        File.join "$(pwd)", host_path
+      end
+    end
+end
diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb
index a1c08379..08e64e06 100644
--- a/test/cli/app_test.rb
+++ b/test/cli/app_test.rb
@@ -55,8 +55,6 @@ class CliAppTest < CliTestCase
   end
 
   test "boot errors leave lock in place" do
-    invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999" }
-
     Kamal::Cli::App.any_instance.expects(:using_version).raises(RuntimeError)
 
     assert !KAMAL.holding_lock?
@@ -66,6 +64,34 @@ class CliAppTest < CliTestCase
     assert KAMAL.holding_lock?
   end
 
+  test "boot with assets" do
+    Object.any_instance.stubs(:sleep)
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
+      .returns("12345678") # running version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
+      .returns("running") # health check
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=web", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
+      .returns("123").twice # old version
+
+    SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
+      .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false)
+      .returns("") # old version
+
+    run_command("boot", config: :with_assets).tap do |output|
+      assert_match "docker tag dhh/app:latest dhh/app:latest", output
+      assert_match "/usr/bin/env mkdir -p .kamal/assets/volumes/app-web-latest ; cp -rn .kamal/assets/extracted/app-web-latest/* .kamal/assets/volumes/app-web-latest ; cp -rn .kamal/assets/extracted/app-web-latest/* .kamal/assets/volumes/app-web-123 || true ; cp -rn .kamal/assets/extracted/app-web-123/* .kamal/assets/volumes/app-web-latest || true", output
+      assert_match "/usr/bin/env mkdir -p .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets 2> /dev/null || true && docker run --name app-web-assets --detach --rm dhh/app:latest sleep 1000000 && docker cp -L app-web-assets:/public/assets/. .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets", output
+      assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} /, output
+      assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
+      assert_match "/usr/bin/env find .kamal/assets/extracted -maxdepth 1 -name 'app-web-*' ! -name app-web-latest -exec rm -rf \"{}\" + ; find .kamal/assets/volumes -maxdepth 1 -name 'app-web-*' ! -name app-web-latest -exec rm -rf \"{}\" +", output
+    end
+  end
+
   test "start" do
     run_command("start").tap do |output|
       assert_match "docker start app-web-999", output
diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb
index c2e020ec..70550ded 100644
--- a/test/commands/app_test.rb
+++ b/test/commands/app_test.rb
@@ -345,8 +345,39 @@ class CommandsAppTest < ActiveSupport::TestCase
     assert_equal "rm -r corddir", new_command.cut_cord("corddir").join(" ")
   end
 
+  test "extract assets" do
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/extracted/app-web-999", "&&",
+      :docker, :stop, "-t 1", "app-web-assets", "2> /dev/null", "|| true", "&&",
+      :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "dhh/app:latest", "sleep 1000000", "&&",
+      :docker, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/assets/extracted/app-web-999", "&&",
+      :docker, :stop, "-t 1", "app-web-assets"
+    ], new_command(asset_path: "/public/assets").extract_assets
+  end
+
+  test "sync asset volumes" do
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rn", ".kamal/assets/extracted/app-web-999/*", ".kamal/assets/volumes/app-web-999"
+    ], new_command(asset_path: "/public/assets").sync_asset_volumes
+
+    assert_equal [
+      :mkdir, "-p", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rn", ".kamal/assets/extracted/app-web-999/*", ".kamal/assets/volumes/app-web-999", ";",
+      :cp, "-rn", ".kamal/assets/extracted/app-web-999/*", ".kamal/assets/volumes/app-web-998", "|| true", ";",
+      :cp, "-rn", ".kamal/assets/extracted/app-web-998/*", ".kamal/assets/volumes/app-web-999", "|| true",
+    ], new_command(asset_path: "/public/assets").sync_asset_volumes(old_version: 998)
+  end
+
+  test "clean up assets" do
+    assert_equal [
+      :find, ".kamal/assets/extracted", "-maxdepth 1", "-name", "'app-web-*'", "!", "-name", "app-web-999", "-exec rm -rf \"{}\" +", ";",
+      :find, ".kamal/assets/volumes", "-maxdepth 1", "-name", "'app-web-*'", "!", "-name", "app-web-999", "-exec rm -rf \"{}\" +"
+    ], new_command(asset_path: "/public/assets").clean_up_assets
+  end
+
   private
-    def new_command(role: "web")
-      Kamal::Commands::App.new(Kamal::Configuration.new(@config, destination: @destination, version: "999"), role: role)
+    def new_command(role: "web", **additional_config)
+      Kamal::Commands::App.new(Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999"), role: role)
     end
 end
diff --git a/test/configuration/role_test.rb b/test/configuration/role_test.rb
index 65003979..49396b74 100644
--- a/test/configuration/role_test.rb
+++ b/test/configuration/role_test.rb
@@ -80,6 +80,15 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     assert_equal expected_env, @config_with_roles.role(:workers).env_file
   end
 
+  test "container name" do
+    ENV["VERSION"] = "12345"
+
+    assert_equal "app-workers-12345", @config_with_roles.role(:workers).container_name
+    assert_equal "app-web-12345", @config_with_roles.role(:web).container_name
+  ensure
+    ENV.delete("VERSION")
+  end
+
   test "env args" do
     assert_equal ["--env-file", ".kamal/env/roles/app-workers.env"], @config_with_roles.role(:workers).env_args
   end
@@ -180,19 +189,67 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
     assert !@config_with_roles.role(:workers).uses_cord?
   end
 
-  test "cord host directory" do
-    assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}}, @config_with_roles.role(:web).cord_host_directory
-  end
-
   test "cord host file" do
-    assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}/cord}, @config_with_roles.role(:web).cord_host_file
+    assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}/cord}, @config_with_roles.role(:web).cord_host_file
   end
 
-  test "cord container directory" do
-    assert_equal "/tmp/kamal-cord", @config_with_roles.role(:web).cord_container_directory
+  test "cord volume" do
+    assert_equal "/tmp/kamal-cord", @config_with_roles.role(:web).cord_volume.container_path
+    assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}}, @config_with_roles.role(:web).cord_volume.host_path
+    assert_equal "--volume", @config_with_roles.role(:web).cord_volume.docker_args[0]
+    assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}:/tmp/kamal-cord}, @config_with_roles.role(:web).cord_volume.docker_args[1]
   end
 
   test "cord container file" do
     assert_equal "/tmp/kamal-cord/cord", @config_with_roles.role(:web).cord_container_file
   end
+
+  test "asset path and volume args" do
+    ENV["VERSION"] = "12345"
+    assert_nil @config_with_roles.role(:web).asset_volume_args
+    assert_nil @config_with_roles.role(:workers).asset_volume_args
+    assert_nil @config_with_roles.role(:web).asset_path
+    assert_nil @config_with_roles.role(:workers).asset_path
+    assert !@config_with_roles.role(:web).assets?
+    assert !@config_with_roles.role(:workers).assets?
+
+    config_with_assets = Kamal::Configuration.new(@deploy_with_roles.dup.tap { |c|
+      c[:asset_path] = "foo"
+    })
+    assert_equal "foo", config_with_assets.role(:web).asset_path
+    assert_equal "foo", config_with_assets.role(:workers).asset_path
+    assert_equal ["--volume", "$(pwd)/.kamal/assets/volumes/app-web-12345:foo"], config_with_assets.role(:web).asset_volume_args
+    assert_nil config_with_assets.role(:workers).asset_volume_args
+    assert config_with_assets.role(:web).assets?
+    assert !config_with_assets.role(:workers).assets?
+
+    config_with_assets = Kamal::Configuration.new(@deploy_with_roles.dup.tap { |c|
+      c[:servers]["web"] = { "hosts" => [ "1.1.1.1", "1.1.1.2" ], "asset_path" => "bar" }
+    })
+    assert_equal "bar", config_with_assets.role(:web).asset_path
+    assert_nil config_with_assets.role(:workers).asset_path
+    assert_equal ["--volume", "$(pwd)/.kamal/assets/volumes/app-web-12345:bar"], config_with_assets.role(:web).asset_volume_args
+    assert_nil config_with_assets.role(:workers).asset_volume_args
+    assert config_with_assets.role(:web).assets?
+    assert !config_with_assets.role(:workers).assets?
+
+  ensure
+    ENV.delete("VERSION")
+  end
+
+  test "asset extracted path" do
+    ENV["VERSION"] = "12345"
+    assert_equal ".kamal/assets/extracted/app-web-12345", @config_with_roles.role(:web).asset_extracted_path
+    assert_equal ".kamal/assets/extracted/app-workers-12345", @config_with_roles.role(:workers).asset_extracted_path
+  ensure
+    ENV.delete("VERSION")
+  end
+
+  test "asset volume path" do
+    ENV["VERSION"] = "12345"
+    assert_equal ".kamal/assets/volumes/app-web-12345", @config_with_roles.role(:web).asset_volume_path
+    assert_equal ".kamal/assets/volumes/app-workers-12345", @config_with_roles.role(:workers).asset_volume_path
+  ensure
+    ENV.delete("VERSION")
+  end
 end
diff --git a/test/configuration/volume_test.rb b/test/configuration/volume_test.rb
new file mode 100644
index 00000000..cea20784
--- /dev/null
+++ b/test/configuration/volume_test.rb
@@ -0,0 +1,13 @@
+require "test_helper"
+
+class ConfigurationVolumeTest < ActiveSupport::TestCase
+  test "docker args absolute" do
+    volume = Kamal::Configuration::Volume.new(host_path: "/root/foo/bar", container_path: "/assets")
+    assert_equal ["--volume", "/root/foo/bar:/assets"], volume.docker_args
+  end
+
+  test "docker args relative" do
+    volume = Kamal::Configuration::Volume.new(host_path: "foo/bar", container_path: "/assets")
+    assert_equal ["--volume", "$(pwd)/foo/bar:/assets"], volume.docker_args
+  end
+end
diff --git a/test/configuration_test.rb b/test/configuration_test.rb
index 59046760..4ed27da5 100644
--- a/test/configuration_test.rb
+++ b/test/configuration_test.rb
@@ -265,4 +265,9 @@ class ConfigurationTest < ActiveSupport::TestCase
     SecureRandom.expects(:hex).with(16).returns("09876543211234567890098765432112")
     assert_equal "09876543211234567890098765432112", @config.run_id
   end
+
+  test "asset path" do
+    assert_nil @config.asset_path
+    assert_equal "foo", Kamal::Configuration.new(@deploy.merge!(asset_path: "foo")).asset_path
+  end
 end
diff --git a/test/fixtures/deploy_with_assets.yml b/test/fixtures/deploy_with_assets.yml
new file mode 100644
index 00000000..0b6b7cf5
--- /dev/null
+++ b/test/fixtures/deploy_with_assets.yml
@@ -0,0 +1,9 @@
+service: app
+image: dhh/app
+servers:
+  - "1.1.1.1"
+  - "1.1.1.2"
+registry:
+  username: user
+  password: pw
+asset_path: /public/assets
diff --git a/test/integration/docker/deployer/app/Dockerfile b/test/integration/docker/deployer/app/Dockerfile
index cbe60ae0..2310ee96 100644
--- a/test/integration/docker/deployer/app/Dockerfile
+++ b/test/integration/docker/deployer/app/Dockerfile
@@ -4,4 +4,5 @@ COPY default.conf /etc/nginx/conf.d/default.conf
 
 ARG COMMIT_SHA
 RUN echo $COMMIT_SHA > /usr/share/nginx/html/version
+RUN mkdir -p /usr/share/nginx/html/versions && echo "version" > /usr/share/nginx/html/versions/$COMMIT_SHA
 
diff --git a/test/integration/docker/deployer/app/config/deploy.yml b/test/integration/docker/deployer/app/config/deploy.yml
index fe4a26ff..ec070f6c 100644
--- a/test/integration/docker/deployer/app/config/deploy.yml
+++ b/test/integration/docker/deployer/app/config/deploy.yml
@@ -8,6 +8,7 @@ env:
     CLEAR_TOKEN: '4321'
   secret:
     - SECRET_TOKEN
+asset_path: /usr/share/nginx/html/versions
 
 registry:
   server: registry:4443
diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb
index e10d9e52..3e0f39e5 100644
--- a/test/integration/main_test.rb
+++ b/test/integration/main_test.rb
@@ -20,6 +20,8 @@ class MainTest < IntegrationTest
     assert_app_is_up version: second_version
     assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy"
 
+    assert_accumulated_assets first_version, second_version
+
     kamal :rollback, first_version
     assert_hooks_ran "pre-connect", "pre-deploy", "post-deploy"
     assert_app_is_up version: first_version
@@ -69,4 +71,10 @@ class MainTest < IntegrationTest
     def assert_no_remote_env_file
       assert_equal "nofile", docker_compose("exec vm1 stat /root/.kamal/env/roles/app-web.env 2> /dev/null || echo nofile", capture: true)
     end
+
+    def assert_accumulated_assets(*versions)
+      versions.each do |version|
+        assert_equal "200", Net::HTTP.get_response(URI.parse("http://localhost:12345/versions/#{version}")).code
+      end
+    end
 end