diff --git a/bin/docs b/bin/docs index a8731ce2..437c5ce7 100755 --- a/bin/docs +++ b/bin/docs @@ -24,6 +24,7 @@ DOCS = { "env" => "Environment variables", "healthcheck" => "Healthchecks", "logging" => "Logging", + "proxy" => "Proxy (Experimental)", "registry" => "Docker Registry", "role" => "Roles", "servers" => "Servers", diff --git a/lib/kamal/cli.rb b/lib/kamal/cli.rb index 6772556e..dc35c403 100644 --- a/lib/kamal/cli.rb +++ b/lib/kamal/cli.rb @@ -1,4 +1,5 @@ module Kamal::Cli + class BootError < StandardError; end class HookError < StandardError; end class LockError < StandardError; end end diff --git a/lib/kamal/cli/app.rb b/lib/kamal/cli/app.rb index ce21b2cf..732addae 100644 --- a/lib/kamal/cli/app.rb +++ b/lib/kamal/cli/app.rb @@ -38,8 +38,17 @@ class Kamal::Cli::App < Kamal::Cli::Base roles = KAMAL.roles_on(host) roles.each do |role| + app = KAMAL.app(role: role, host: host) execute *KAMAL.auditor.record("Started app version #{KAMAL.config.version}"), verbosity: :debug - execute *KAMAL.app(role: role, host: host).start, raise_on_non_zero_exit: false + execute *app.start, raise_on_non_zero_exit: false + + if role.running_traefik? && KAMAL.proxy_host?(host) + version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip + endpoint = capture_with_info(*app.container_endpoint(version: version)).strip + raise Kamal::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty? + + execute *KAMAL.proxy.deploy(role.container_prefix, target: endpoint) + end end end end @@ -52,8 +61,18 @@ class Kamal::Cli::App < Kamal::Cli::Base roles = KAMAL.roles_on(host) roles.each do |role| + app = KAMAL.app(role: role, host: host) execute *KAMAL.auditor.record("Stopped app", role: role), verbosity: :debug - execute *KAMAL.app(role: role, host: host).stop, raise_on_non_zero_exit: false + + if role.running_traefik? && KAMAL.proxy_host?(host) + version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip + endpoint = capture_with_info(*app.container_endpoint(version: version)).strip + if endpoint.present? + execute *KAMAL.proxy.remove(role.container_prefix, target: endpoint), raise_on_non_zero_exit: false + end + end + + execute *app.stop, raise_on_non_zero_exit: false end end end diff --git a/lib/kamal/cli/app/boot.rb b/lib/kamal/cli/app/boot.rb index df3e6925..b6a09477 100644 --- a/lib/kamal/cli/app/boot.rb +++ b/lib/kamal/cli/app/boot.rb @@ -45,15 +45,25 @@ class Kamal::Cli::App::Boot def start_new_version audit "Booted app version #{version}" - - execute *app.tie_cord(role.cord_host_file) if uses_cord? hostname = "#{host.to_s[0...51].gsub(/\.+$/, '')}-#{SecureRandom.hex(6)}" execute *app.ensure_env_directory upload! role.secrets_io(host), role.secrets_path, mode: "0600" - execute *app.run(hostname: hostname) - Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) } + if proxy_host? + execute *app.run_for_proxy(hostname: hostname) + if running_traefik? + endpoint = capture_with_info(*app.container_endpoint(version: version)).strip + raise Kamal::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty? + execute *KAMAL.proxy.deploy(role.container_prefix, target: endpoint) + else + Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) } + end + else + execute *app.tie_cord(role.cord_host_file) if uses_cord? + execute *app.run(hostname: hostname) + Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) } + end end def stop_new_version @@ -61,7 +71,7 @@ class Kamal::Cli::App::Boot end def stop_old_version(version) - if uses_cord? + if uses_cord? && !proxy_host? cord = capture_with_info(*app.cord(version: version), raise_on_non_zero_exit: false).strip if cord.present? execute *app.cut_cord(cord) @@ -124,4 +134,8 @@ class Kamal::Cli::App::Boot def queuer? barrier && !barrier_role? end + + def proxy_host? + KAMAL.proxy_host?(host) + end end diff --git a/lib/kamal/cli/main.rb b/lib/kamal/cli/main.rb index 3bd6dc24..fdbc2559 100644 --- a/lib/kamal/cli/main.rb +++ b/lib/kamal/cli/main.rb @@ -35,8 +35,13 @@ class Kamal::Cli::Main < Kamal::Cli::Base with_lock do run_hook "pre-deploy", secrets: true - say "Ensure Traefik is running...", :magenta - invoke "kamal:cli:traefik:boot", [], invoke_options + if KAMAL.config.proxy.enabled? + say "Ensure Traefik/kamal-proxy is running...", :magenta + invoke "kamal:cli:proxy:boot", [], invoke_options + else + say "Ensure Traefik is running...", :magenta + invoke "kamal:cli:traefik:boot", [], invoke_options + end say "Detect stale containers...", :magenta invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true) @@ -104,7 +109,11 @@ class Kamal::Cli::Main < Kamal::Cli::Base desc "details", "Show details about all containers" def details - invoke "kamal:cli:traefik:details" + if KAMAL.config.proxy.enabled? + invoke "kamal:cli:proxy:details" + else + invoke "kamal:cli:traefik:details" + end invoke "kamal:cli:app:details" invoke "kamal:cli:accessory:details", [ "all" ] end @@ -181,7 +190,11 @@ class Kamal::Cli::Main < Kamal::Cli::Base def remove confirming "This will remove all containers and images. Are you sure?" do with_lock do - invoke "kamal:cli:traefik:remove", [], options.without(:confirmed) + if KAMAL.config.proxy.enabled? + invoke "kamal:cli:proxy:remove", [], options.without(:confirmed) + else + invoke "kamal:cli:traefik:remove", [], options.without(:confirmed) + end invoke "kamal:cli:app:remove", [], options.without(:confirmed) invoke "kamal:cli:accessory:remove", [ "all" ], options invoke "kamal:cli:registry:logout", [], options.without(:confirmed).merge(skip_local: true) @@ -206,6 +219,9 @@ class Kamal::Cli::Main < Kamal::Cli::Base desc "lock", "Manage the deploy lock" subcommand "lock", Kamal::Cli::Lock + desc "proxy", "Prune old application images and containers" + subcommand "proxy", Kamal::Cli::Proxy + desc "prune", "Prune old application images and containers" subcommand "prune", Kamal::Cli::Prune diff --git a/lib/kamal/cli/proxy.rb b/lib/kamal/cli/proxy.rb new file mode 100644 index 00000000..d546439c --- /dev/null +++ b/lib/kamal/cli/proxy.rb @@ -0,0 +1,160 @@ +class Kamal::Cli::Proxy < Kamal::Cli::Base + desc "boot", "Boot proxy on servers" + def boot + raise_unless_kamal_proxy_enabled! + with_lock do + on(KAMAL.traefik_hosts) do |host| + execute *KAMAL.registry.login + execute *KAMAL.traefik_or_proxy(host).start_or_run + end + end + end + + desc "reboot", "Reboot proxy on servers (stop container, remove container, start new container)" + option :rolling, type: :boolean, default: false, desc: "Reboot proxy on hosts in sequence, rather than in parallel" + option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question" + def reboot + raise_unless_kamal_proxy_enabled! + confirming "This will cause a brief outage on each host. Are you sure?" do + with_lock do + host_groups = options[:rolling] ? KAMAL.traefik_hosts : [ KAMAL.traefik_hosts ] + host_groups.each do |hosts| + host_list = Array(hosts).join(",") + run_hook "pre-traefik-reboot", hosts: host_list + on(hosts) do |host| + execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug + execute *KAMAL.registry.login + + "Stopping and removing Traefik on #{host}, if running..." + execute *KAMAL.traefik.stop, raise_on_non_zero_exit: false + execute *KAMAL.traefik.remove_container + + "Stopping and removing kamal-proxy on #{host}, if running..." + execute *KAMAL.proxy.stop, raise_on_non_zero_exit: false + execute *KAMAL.proxy.remove_container + + execute *KAMAL.traefik_or_proxy(host).run + + if KAMAL.proxy_host?(host) + KAMAL.roles_on(host).select(&:running_traefik?).each do |role| + app = KAMAL.app(role: role, host: host) + + version = capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip + endpoint = capture_with_info(*app.container_endpoint(version: version)).strip + + if endpoint.present? + info "Deploying #{endpoint} for role `#{role}` on #{host}..." + execute *KAMAL.proxy.deploy(role.container_prefix, target: endpoint) + end + end + end + end + run_hook "post-traefik-reboot", hosts: host_list + end + end + end + end + + desc "start", "Start existing proxy container on servers" + def start + raise_unless_kamal_proxy_enabled! + with_lock do + on(KAMAL.traefik_hosts) do |host| + execute *KAMAL.auditor.record("Started proxy"), verbosity: :debug + execute *KAMAL.traefik_or_proxy(host).start + end + end + end + + desc "stop", "Stop existing proxy container on servers" + def stop + raise_unless_kamal_proxy_enabled! + with_lock do + on(KAMAL.traefik_hosts) do |host| + execute *KAMAL.auditor.record("Stopped proxy"), verbosity: :debug + execute *KAMAL.traefik_or_proxy(host).stop, raise_on_non_zero_exit: false + end + end + end + + desc "restart", "Restart existing proxy container on servers" + def restart + raise_unless_kamal_proxy_enabled! + with_lock do + stop + start + end + end + + desc "details", "Show details about proxy container from servers" + def details + raise_unless_kamal_proxy_enabled! + on(KAMAL.traefik_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.traefik_or_proxy(host).info), type: "Proxy" } + end + + desc "logs", "Show log lines from proxy on servers" + option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)" + option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server" + option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)" + option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)" + def logs + raise_unless_kamal_proxy_enabled! + grep = options[:grep] + + if options[:follow] + run_locally do + info "Following logs on #{KAMAL.primary_host}..." + info KAMAL.traefik_or_proxy(KAMAL.primary_host).follow_logs(host: KAMAL.primary_host, grep: grep) + exec KAMAL.traefik_or_proxy(KAMAL.primary_host).follow_logs(host: KAMAL.primary_host, grep: grep) + end + else + since = options[:since] + lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set + + on(KAMAL.traefik_hosts) do |host| + puts_by_host host, capture(*KAMAL.traefik_or_proxy(host).logs(since: since, lines: lines, grep: grep)), type: "Proxy" + end + end + end + + desc "remove", "Remove proxy container and image from servers" + def remove + raise_unless_kamal_proxy_enabled! + with_lock do + stop + remove_container + remove_image + end + end + + desc "remove_container", "Remove proxy container from servers", hide: true + def remove_container + raise_unless_kamal_proxy_enabled! + with_lock do + on(KAMAL.traefik_hosts) do + execute *KAMAL.auditor.record("Removed proxy container"), verbosity: :debug + execute *KAMAL.proxy.remove_container + execute *KAMAL.traefik.remove_container + end + end + end + + desc "remove_image", "Remove proxy image from servers", hide: true + def remove_image + raise_unless_kamal_proxy_enabled! + with_lock do + on(KAMAL.traefik_hosts) do + execute *KAMAL.auditor.record("Removed proxy image"), verbosity: :debug + execute *KAMAL.proxy.remove_image + execute *KAMAL.traefik.remove_image + end + end + end + + private + def raise_unless_kamal_proxy_enabled! + unless KAMAL.config.proxy.enabled? + raise "kamal proxy commands are disabled unless experimental proxy support is enabled. Use `kamal traefik` commands instead." + end + end +end diff --git a/lib/kamal/cli/traefik.rb b/lib/kamal/cli/traefik.rb index 41ffbc04..0dc25955 100644 --- a/lib/kamal/cli/traefik.rb +++ b/lib/kamal/cli/traefik.rb @@ -1,6 +1,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base desc "boot", "Boot Traefik on servers" def boot + raise_if_kamal_proxy_enabled! with_lock do on(KAMAL.traefik_hosts) do execute *KAMAL.registry.login @@ -15,6 +16,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base option :rolling, type: :boolean, default: false, desc: "Reboot traefik on hosts in sequence, rather than in parallel" option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question" def reboot + raise_if_kamal_proxy_enabled! confirming "This will cause a brief outage on each host. Are you sure?" do with_lock do host_groups = options[:rolling] ? KAMAL.traefik_hosts : [ KAMAL.traefik_hosts ] @@ -36,6 +38,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base desc "start", "Start existing Traefik container on servers" def start + raise_if_kamal_proxy_enabled! with_lock do on(KAMAL.traefik_hosts) do execute *KAMAL.auditor.record("Started traefik"), verbosity: :debug @@ -46,6 +49,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base desc "stop", "Stop existing Traefik container on servers" def stop + raise_if_kamal_proxy_enabled! with_lock do on(KAMAL.traefik_hosts) do execute *KAMAL.auditor.record("Stopped traefik"), verbosity: :debug @@ -56,6 +60,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base desc "restart", "Restart existing Traefik container on servers" def restart + raise_if_kamal_proxy_enabled! with_lock do stop start @@ -64,6 +69,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base desc "details", "Show details about Traefik container from servers" def details + raise_if_kamal_proxy_enabled! on(KAMAL.traefik_hosts) { |host| puts_by_host host, capture_with_info(*KAMAL.traefik.info), type: "Traefik" } end @@ -74,6 +80,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base option :grep_options, aliases: "-o", desc: "Additional options supplied to grep" option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)" def logs + raise_if_kamal_proxy_enabled! grep = options[:grep] grep_options = options[:grep_options] @@ -95,6 +102,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base desc "remove", "Remove Traefik container and image from servers" def remove + raise_if_kamal_proxy_enabled! with_lock do stop remove_container @@ -104,6 +112,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base desc "remove_container", "Remove Traefik container from servers", hide: true def remove_container + raise_if_kamal_proxy_enabled! with_lock do on(KAMAL.traefik_hosts) do execute *KAMAL.auditor.record("Removed traefik container"), verbosity: :debug @@ -114,6 +123,7 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base desc "remove_image", "Remove Traefik image from servers", hide: true def remove_image + raise_if_kamal_proxy_enabled! with_lock do on(KAMAL.traefik_hosts) do execute *KAMAL.auditor.record("Removed traefik image"), verbosity: :debug @@ -121,4 +131,11 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base end end end + + private + def raise_if_kamal_proxy_enabled! + if KAMAL.config.proxy.enabled? + raise "kamal traefik commands are disabled when experimental proxy support is enabled. Use `kamal proxy` commands instead." + end + end end diff --git a/lib/kamal/commander.rb b/lib/kamal/commander.rb index 11914a67..994debb5 100644 --- a/lib/kamal/commander.rb +++ b/lib/kamal/commander.rb @@ -4,7 +4,7 @@ require "active_support/core_ext/object/blank" class Kamal::Commander attr_accessor :verbosity, :holding_lock, :connected - delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :traefik_hosts, :accessory_hosts, to: :specifics + delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :traefik_hosts, :proxy_hosts, :proxy_host?, :accessory_hosts, to: :specifics def initialize self.verbosity = :info @@ -106,6 +106,10 @@ class Kamal::Commander @lock ||= Kamal::Commands::Lock.new(config) end + def proxy + @proxy ||= Kamal::Commands::Proxy.new(config) + end + def prune @prune ||= Kamal::Commands::Prune.new(config) end @@ -127,6 +131,10 @@ class Kamal::Commander end + def traefik_or_proxy(host) + proxy_host?(host) ? proxy : traefik + end + def with_verbosity(level) old_level = self.verbosity diff --git a/lib/kamal/commander/specifics.rb b/lib/kamal/commander/specifics.rb index 127bd40e..12a710d8 100644 --- a/lib/kamal/commander/specifics.rb +++ b/lib/kamal/commander/specifics.rb @@ -22,6 +22,15 @@ class Kamal::Commander::Specifics config.traefik_hosts & specified_hosts end + def proxy_hosts + traefik_hosts & config.proxy_hosts + end + + def proxy_host?(host) + host = host.hostname if host.is_a?(SSHKit::Host) + proxy_hosts.include?(host) + end + def accessory_hosts specific_hosts || config.accessories.flat_map(&:hosts) end diff --git a/lib/kamal/commands/app.rb b/lib/kamal/commands/app.rb index f1991e48..ad6e0613 100644 --- a/lib/kamal/commands/app.rb +++ b/lib/kamal/commands/app.rb @@ -30,6 +30,24 @@ class Kamal::Commands::App < Kamal::Commands::Base role.cmd end + def run_for_proxy(hostname: nil) + docker :run, + "--detach", + "--restart unless-stopped", + "--name", container_name, + *([ "--hostname", hostname ] if hostname), + "-e", "KAMAL_CONTAINER_NAME=\"#{container_name}\"", + "-e", "KAMAL_VERSION=\"#{config.version}\"", + *role.env_args(host), + *role.logging_args, + *config.volume_args, + *role.asset_volume_args, + *role.label_args_for_proxy, + *role.option_args, + config.absolute_image, + role.cmd + end + def start docker :start, container_name end diff --git a/lib/kamal/commands/app/containers.rb b/lib/kamal/commands/app/containers.rb index 0bab388b..629fa47f 100644 --- a/lib/kamal/commands/app/containers.rb +++ b/lib/kamal/commands/app/containers.rb @@ -28,4 +28,11 @@ module Kamal::Commands::App::Containers container_id_for(container_name: container_name(version)), xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT)) end + + def container_endpoint(version:) + pipe \ + container_id_for(container_name: container_name(version)), + xargs(docker(:inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'")), + [ :sed, "-e", "'s/\\/tcp$//'" ] + end end diff --git a/lib/kamal/commands/proxy.rb b/lib/kamal/commands/proxy.rb new file mode 100644 index 00000000..89808d72 --- /dev/null +++ b/lib/kamal/commands/proxy.rb @@ -0,0 +1,69 @@ +class Kamal::Commands::Proxy < Kamal::Commands::Base + delegate :argumentize, :optionize, to: Kamal::Utils + delegate :container_name, to: :proxy_config + + attr_reader :proxy_config + + def initialize(config) + super + @proxy_config = config.proxy + end + + def run + docker :run, + "--name", container_name, + "--detach", + "--restart", "unless-stopped", + *proxy_config.publish_args, + "--volume", "/var/run/docker.sock:/var/run/docker.sock", + "--volume", "#{container_name}:/root/.config/kamal-proxy", + *config.logging_args, + proxy_config.image + end + + def start + docker :container, :start, container_name + end + + def stop(name: container_name) + docker :container, :stop, name + end + + def start_or_run + combine start, run, by: "||" + end + + def deploy(service, target:) + optionize({ target: target }) + docker :exec, container_name, "kamal-proxy", :deploy, service, *optionize({ target: target }), *proxy_config.deploy_command_args + end + + def remove(service, target:) + docker :exec, container_name, "kamal-proxy", :remove, service, *optionize({ target: target }) + end + + def info + docker :ps, "--filter", "name=^#{container_name}$" + end + + def logs(since: nil, lines: nil, grep: nil, grep_options: nil) + pipe \ + docker(:logs, container_name, (" --since #{since}" if since), (" --tail #{lines}" if lines), "--timestamps", "2>&1"), + ("grep '#{grep}'#{" #{grep_options}" if grep_options}" if grep) + end + + def follow_logs(host:, grep: nil, grep_options: nil) + run_over_ssh pipe( + docker(:logs, container_name, "--timestamps", "--tail", "10", "--follow", "2>&1"), + (%(grep "#{grep}"#{" #{grep_options}" if grep_options}) if grep) + ).join(" "), host: host + end + + def remove_container + docker :container, :prune, "--force", "--filter", "label=org.opencontainers.image.title=kamal-proxy" + end + + def remove_image + docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=kamal-proxy" + end +end diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index 0194bdd2..30ff7af1 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -10,7 +10,7 @@ class Kamal::Configuration delegate :argumentize, :optionize, to: Kamal::Utils attr_reader :destination, :raw_config - attr_reader :accessories, :aliases, :boot, :builder, :env, :healthcheck, :logging, :traefik, :servers, :ssh, :sshkit, :registry + attr_reader :accessories, :aliases :boot, :builder, :env, :healthcheck, :logging, :proxy, :traefik, :servers, :ssh, :sshkit, :registry include Validation @@ -60,6 +60,7 @@ class Kamal::Configuration @healthcheck = Healthcheck.new(healthcheck_config: @raw_config.healthcheck) @logging = Logging.new(logging_config: @raw_config.logging) + @proxy = Proxy.new(config: self) @traefik = Traefik.new(config: self) @ssh = Ssh.new(config: self) @sshkit = Sshkit.new(config: self) @@ -141,6 +142,10 @@ class Kamal::Configuration traefik_roles.flat_map(&:hosts).uniq end + def proxy_hosts + proxy.hosts + end + def repository [ registry.server, image ].compact.join("/") end diff --git a/lib/kamal/configuration/docs/configuration.yml b/lib/kamal/configuration/docs/configuration.yml index f1045dd6..8ebaaa0d 100644 --- a/lib/kamal/configuration/docs/configuration.yml +++ b/lib/kamal/configuration/docs/configuration.yml @@ -143,6 +143,12 @@ accessories: traefik: ... +# Proxy +# +# **Experimental** Configuration for kamal-proxy the replacement for Traefik, see kamal docs proxy +proxy: + ... + # SSHKit # # See kamal docs sshkit diff --git a/lib/kamal/configuration/docs/proxy.yml b/lib/kamal/configuration/docs/proxy.yml new file mode 100644 index 00000000..f61ea229 --- /dev/null +++ b/lib/kamal/configuration/docs/proxy.yml @@ -0,0 +1,83 @@ +# Proxy +# +# **Experimental** [kamal-proxy](http://github.com/basecamp/kamal-proxy) is a +# custom built specifically for Kamal. It will replace Traefik in Kamal v2.0, +# but currently is available as an experimental feature. +# +# When this is enabled, the proxy will be started on the hosts listed under the hosts key. +# In addition, the kamal traefik command will be disabled and replaced by kamal proxy. +# +# The kamal proxy command works identically to kamal traefik on hosts that have not +# been included. It will also handle switching between Traefik and kamal-proxy when you +# run kamal proxy reboot. + +# Limitations +# +# Currently the proxy will run on ports 80 and 443 and will bind to those +# ports on the host. +# +# There is no way to set custom options for `docker run` when booting the proxy. +# +# If you have custom Traefik configuration via labels or boot arguments they may +# not have an equivalent in kamal-proxy. + +# Proxy settings +# +# The proxy is configured in the root configuration under `traefik`. These are +# options that are set when deploying the application, not when booting the proxy +# +# They are application specific, so are not shared when multiple applications +# with the same proxy. +proxy: + + # Enabled + # + # Whether to enable experimental proxy support. Defaults to false + enabled: true + + # Hosts + # + # The hosts to run the proxy on, instead of Traefik + # This is a temporary setting and will be removed when we full switch to kamal-proxy + # + # If you run `kamal traefik reboot`, then the proxy will be started on these hosts + # in place of traefik. + hosts: + - 10.0.0.1 + - 10.0.0.2 + + # Host + # + # This is the host that will be used to serve the app. By setting this you can run + # multiple apps on the same server sharing the same instance of the proxy. + # + # If this is set only requests that match this host will be forwarded by the proxy. + # if this is not set, then all requests will be forwarded, except for matching + # requests for other apps that do have a host set. + host: foo.example.com + + # Deploy timeout + # + # How long to wait for the app to boot when deploying, defaults to 30 seconds + deploy_timeout: 10s + + # Response timeout + # + # How long to wait for requests to complete before timing out, defaults to 30 seconds + response_timeout: 10 + + # Healthcheck + # + # When deploying, the proxy will by default hit /up once every second until we hit + # the deploy timeout, with a 5 second timeout for each request. + # + # Once the app is up, the proxy will stop hitting the healthcheck endpoint. + healthcheck: + interval: 3 + path: /health + timeout: 3 + + # Max Request Body Size + # + # The maximum request size in bytes that the proxy will accept, defaults to 1GB + max_request_body_size: 40_000_000 diff --git a/lib/kamal/configuration/proxy.rb b/lib/kamal/configuration/proxy.rb new file mode 100644 index 00000000..5dc3546b --- /dev/null +++ b/lib/kamal/configuration/proxy.rb @@ -0,0 +1,57 @@ +class Kamal::Configuration::Proxy + include Kamal::Configuration::Validation + + DEFAULT_HTTP_PORT = 80 + DEFAULT_HTTPS_PORT = 443 + DEFAULT_IMAGE = "basecamp/kamal-proxy:latest" + + delegate :argumentize, :optionize, to: Kamal::Utils + + def initialize(config:) + @proxy_config = config.raw_config.proxy || {} + validate! proxy_config + end + + def enabled? + !!proxy_config.fetch("enabled", false) + end + + def hosts + if enabled? + proxy_config.fetch("hosts", []) + else + [] + end + end + + def image + proxy_config.fetch("image", DEFAULT_IMAGE) + end + + def container_name + "kamal-proxy" + end + + def publish_args + argumentize "--publish", [ "#{DEFAULT_HTTP_PORT}:#{DEFAULT_HTTP_PORT}", "#{DEFAULT_HTTPS_PORT}:#{DEFAULT_HTTPS_PORT}" ] + end + + def deploy_options + { + host: proxy_config["host"], + "deploy-timeout": proxy_config["deploy_timeout"], + "drain-timeout": proxy_config["drain_timeout"], + "health-check-interval": proxy_config.dig("health_check", "interval"), + "health-check-timeout": proxy_config.dig("health_check", "timeout"), + "health-check-path": proxy_config.dig("health_check", "path"), + "target-timeout": proxy_config["response_timeout"] + }.compact + end + + def deploy_command_args + optionize deploy_options + end + + private + attr_accessor :proxy_config +end diff --git a/lib/kamal/configuration/role.rb b/lib/kamal/configuration/role.rb index ef651898..6579b9d0 100644 --- a/lib/kamal/configuration/role.rb +++ b/lib/kamal/configuration/role.rb @@ -58,10 +58,18 @@ class Kamal::Configuration::Role default_labels.merge(traefik_labels).merge(custom_labels) end + def labels_for_proxy + default_labels.merge(custom_labels) + end + def label_args argumentize "--label", labels end + def label_args_for_proxy + argumentize "--label", labels_for_proxy + end + def logging_args logging.args end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 46a067f3..744519a7 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -356,6 +356,18 @@ class CliAppTest < CliTestCase end end + test "boot proxy" do + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + + run_command("boot", config: :with_proxy).tap do |output| + assert_match /Renaming container .* to .* as already deployed on 1.1.1.1/, output # Rename + assert_match /docker rename app-web-latest app-web-latest_replaced_[0-9a-f]{16}/, output + assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} -e KAMAL_CONTAINER_NAME="app-web-latest" -e KAMAL_VERSION="latest" --env-file .kamal\/env\/roles\/app-web.env --log-opt max-size="10m" --label service="app" --label role="web" --label destination dhh\/app:latest/, output + assert_match /docker exec kamal-proxy kamal-proxy deploy app-web --target "123"/, output + assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output + end + end + private def run_command(*command, config: :with_accessories, host: "1.1.1.1", allow_execute_error: false) stdouted do diff --git a/test/cli/proxy_test.rb b/test/cli/proxy_test.rb new file mode 100644 index 00000000..7d08c62f --- /dev/null +++ b/test/cli/proxy_test.rb @@ -0,0 +1,141 @@ +require_relative "cli_test_case" + +class CliProxyTest < CliTestCase + test "boot" do + run_command("boot").tap do |output| + assert_match "docker login", output + assert_match "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}", output + end + end + + test "reboot" do + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'", "|", :sed, "-e", "'s/\\/tcp$//'") + .returns("172.1.0.2:80") + .at_least_once + + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with { |*args| args[0..1] == [ :sh, "-c" ] } + .returns("123") + .at_least_once + + run_command("reboot", "-y").tap do |output| + assert_match "docker container stop kamal-proxy on 1.1.1.1", output + assert_match "docker container stop traefik on 1.1.1.1", output + assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.1", output + assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.1", output + assert_match "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE} on 1.1.1.1", output + assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target \"172.1.0.2:80\" --deploy-timeout \"6s\" on 1.1.1.1", output + + assert_match "docker container stop kamal-proxy on 1.1.1.2", output + assert_match "docker container stop traefik on 1.1.1.2", output + assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.2", output + assert_match "docker container prune --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.2", output + assert_match "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" traefik:v2.10 --providers.docker --log.level=\"DEBUG\" on 1.1.1.2", output + end + end + + test "reboot --rolling" do + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{.NetworkSettings.IPAddress}}{{range $k, $v := .NetworkSettings.Ports}}{{printf \":%s\" $k}}{{break}}{{end}}'", "|", :sed, "-e", "'s/\\/tcp$//'") + .returns("172.1.0.2:80") + .at_least_once + + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with { |*args| args[0..1] == [ :sh, "-c" ] } + .returns("123") + .at_least_once + + run_command("reboot", "--rolling", "-y").tap do |output| + assert_match "Running docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.1", output + end + end + + test "start" do + run_command("start").tap do |output| + assert_match "docker container start kamal-proxy", output + end + end + + test "stop" do + run_command("stop").tap do |output| + assert_match "docker container stop kamal-proxy", output + end + end + + test "restart" do + Kamal::Cli::Proxy.any_instance.expects(:stop) + Kamal::Cli::Proxy.any_instance.expects(:start) + + run_command("restart") + end + + test "details" do + run_command("details").tap do |output| + assert_match "docker ps --filter name=^kamal-proxy$", output + end + end + + test "logs" do + SSHKit::Backend::Abstract.any_instance.stubs(:capture) + .with(:docker, :logs, "kamal-proxy", " --tail 100", "--timestamps", "2>&1") + .returns("Log entry") + + SSHKit::Backend::Abstract.any_instance.stubs(:capture) + .with(:docker, :logs, "traefik", " --tail 100", "--timestamps", "2>&1") + .returns("Log entry") + + run_command("logs").tap do |output| + assert_match "Proxy Host: 1.1.1.1", output + assert_match "Log entry", output + end + end + + test "logs with follow" do + SSHKit::Backend::Abstract.any_instance.stubs(:exec) + .with("ssh -t root@1.1.1.1 -p 22 'docker logs kamal-proxy --timestamps --tail 10 --follow 2>&1'") + + assert_match "docker logs kamal-proxy --timestamps --tail 10 --follow", run_command("logs", "--follow") + end + + test "remove" do + Kamal::Cli::Proxy.any_instance.expects(:stop) + Kamal::Cli::Proxy.any_instance.expects(:remove_container) + Kamal::Cli::Proxy.any_instance.expects(:remove_image) + + run_command("remove") + end + + test "remove_container" do + run_command("remove_container").tap do |output| + assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy", output + end + end + + test "remove_image" do + run_command("remove_image").tap do |output| + assert_match "docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy", output + end + end + + test "commands disallowed when proxy is disabled" do + assert_raises_when_disabled "boot" + assert_raises_when_disabled "reboot" + assert_raises_when_disabled "start" + assert_raises_when_disabled "stop" + assert_raises_when_disabled "details" + assert_raises_when_disabled "logs" + assert_raises_when_disabled "remove" + end + + private + def run_command(*command, fixture: :with_proxy) + stdouted { Kamal::Cli::Proxy.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) } + end + + def assert_raises_when_disabled(command) + assert_raises "kamal proxy commands are disabled unless experimental proxy support is enabled. Use `kamal traefik` commands instead." do + run_command(command, fixture: :with_accessories) + end + end +end diff --git a/test/commands/proxy_test.rb b/test/commands/proxy_test.rb new file mode 100644 index 00000000..73e7bfbf --- /dev/null +++ b/test/commands/proxy_test.rb @@ -0,0 +1,126 @@ +require "test_helper" + +class CommandsProxyTest < ActiveSupport::TestCase + setup do + @config = { + service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ] + } + + ENV["EXAMPLE_API_KEY"] = "456" + end + + teardown do + ENV.delete("EXAMPLE_API_KEY") + end + + test "run" do + assert_equal \ + "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}", + new_command.run.join(" ") + end + + test "run with ports configured" do + assert_equal \ + "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}", + new_command.run.join(" ") + end + + test "run without configuration" do + @config.delete(:proxy) + + assert_equal \ + "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-opt max-size=\"10m\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}", + new_command.run.join(" ") + end + + test "run with logging config" do + @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } + + assert_equal \ + "docker run --name kamal-proxy --detach --restart unless-stopped --publish 80:80 --publish 443:443 --volume /var/run/docker.sock:/var/run/docker.sock --volume kamal-proxy:/root/.config/kamal-proxy --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" #{Kamal::Configuration::Proxy::DEFAULT_IMAGE}", + new_command.run.join(" ") + end + + test "proxy start" do + assert_equal \ + "docker container start kamal-proxy", + new_command.start.join(" ") + end + + test "proxy stop" do + assert_equal \ + "docker container stop kamal-proxy", + new_command.stop.join(" ") + end + + test "proxy info" do + assert_equal \ + "docker ps --filter name=^kamal-proxy$", + new_command.info.join(" ") + end + + test "proxy logs" do + assert_equal \ + "docker logs kamal-proxy --timestamps 2>&1", + new_command.logs.join(" ") + end + + test "proxy logs since 2h" do + assert_equal \ + "docker logs kamal-proxy --since 2h --timestamps 2>&1", + new_command.logs(since: "2h").join(" ") + end + + test "proxy logs last 10 lines" do + assert_equal \ + "docker logs kamal-proxy --tail 10 --timestamps 2>&1", + new_command.logs(lines: 10).join(" ") + end + + test "proxy logs with grep hello!" do + assert_equal \ + "docker logs kamal-proxy --timestamps 2>&1 | grep 'hello!'", + new_command.logs(grep: "hello!").join(" ") + end + + test "proxy remove container" do + assert_equal \ + "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy", + new_command.remove_container.join(" ") + end + + test "proxy remove image" do + assert_equal \ + "docker image prune --all --force --filter label=org.opencontainers.image.title=kamal-proxy", + new_command.remove_image.join(" ") + end + + test "proxy follow logs" do + assert_equal \ + "ssh -t root@1.1.1.1 -p 22 'docker logs kamal-proxy --timestamps --tail 10 --follow 2>&1'", + new_command.follow_logs(host: @config[:servers].first) + end + + test "proxy follow logs with grep hello!" do + assert_equal \ + "ssh -t root@1.1.1.1 -p 22 'docker logs kamal-proxy --timestamps --tail 10 --follow 2>&1 | grep \"hello!\"'", + new_command.follow_logs(host: @config[:servers].first, grep: "hello!") + end + + test "deploy" do + assert_equal \ + "docker exec kamal-proxy kamal-proxy deploy service --target \"172.1.0.2:80\"", + new_command.deploy("service", target: "172.1.0.2:80").join(" ") + end + + test "remove" do + assert_equal \ + "docker exec kamal-proxy kamal-proxy remove service --target \"172.1.0.2:80\"", + new_command.remove("service", target: "172.1.0.2:80").join(" ") + end + + private + def new_command + Kamal::Commands::Proxy.new(Kamal::Configuration.new(@config, version: "123")) + end +end diff --git a/test/fixtures/deploy_with_proxy.yml b/test/fixtures/deploy_with_proxy.yml new file mode 100644 index 00000000..bfe8c505 --- /dev/null +++ b/test/fixtures/deploy_with_proxy.yml @@ -0,0 +1,42 @@ +service: app +image: dhh/app +servers: + web: + - "1.1.1.1" + - "1.1.1.2" + workers: + - "1.1.1.3" + - "1.1.1.4" +registry: + username: user + password: pw + +proxy: + enabled: true + hosts: + - "1.1.1.1" + deploy_timeout: 6s + +accessories: + mysql: + image: mysql:5.7 + host: 1.1.1.3 + port: 3306 + env: + clear: + MYSQL_ROOT_HOST: '%' + secret: + - MYSQL_ROOT_PASSWORD + files: + - test/fixtures/files/my.cnf:/etc/mysql/my.cnf + directories: + - data:/var/lib/mysql + redis: + image: redis:latest + roles: + - web + port: 6379 + directories: + - data:/data + +readiness_delay: 0 diff --git a/test/integration/broken_deploy_test.rb b/test/integration/broken_deploy_test.rb index 77f0ff96..964f1d0b 100644 --- a/test/integration/broken_deploy_test.rb +++ b/test/integration/broken_deploy_test.rb @@ -27,6 +27,5 @@ class BrokenDeployTest < IntegrationTest assert_match /First web container is unhealthy on vm[12], not booting any other roles/, output assert_match "First web container is unhealthy, not booting workers on vm3", output assert_match "nginx: [emerg] unexpected end of file, expecting \";\" or \"}\" in /etc/nginx/conf.d/default.conf:2", output - assert_match 'ERROR {"Status":"unhealthy","FailingStreak":0,"Log":[]}', output end end diff --git a/test/integration/docker/deployer/app/Dockerfile b/test/integration/docker/deployer/app/Dockerfile index dc270aa9..0e6237df 100644 --- a/test/integration/docker/deployer/app/Dockerfile +++ b/test/integration/docker/deployer/app/Dockerfile @@ -6,4 +6,4 @@ ARG COMMIT_SHA RUN echo $COMMIT_SHA > /usr/share/nginx/html/version RUN mkdir -p /usr/share/nginx/html/versions && echo "version" > /usr/share/nginx/html/versions/$COMMIT_SHA RUN mkdir -p /usr/share/nginx/html/versions && echo "hidden" > /usr/share/nginx/html/versions/.hidden - +RUN echo "Up!" > /usr/share/nginx/html/up diff --git a/test/integration/docker/deployer/app_with_roles/Dockerfile b/test/integration/docker/deployer/app_with_roles/Dockerfile index dc270aa9..0e6237df 100644 --- a/test/integration/docker/deployer/app_with_roles/Dockerfile +++ b/test/integration/docker/deployer/app_with_roles/Dockerfile @@ -6,4 +6,4 @@ ARG COMMIT_SHA RUN echo $COMMIT_SHA > /usr/share/nginx/html/version RUN mkdir -p /usr/share/nginx/html/versions && echo "version" > /usr/share/nginx/html/versions/$COMMIT_SHA RUN mkdir -p /usr/share/nginx/html/versions && echo "hidden" > /usr/share/nginx/html/versions/.hidden - +RUN echo "Up!" > /usr/share/nginx/html/up diff --git a/test/integration/docker/deployer/app_with_roles/config/deploy.yml b/test/integration/docker/deployer/app_with_roles/config/deploy.yml index 3b942665..c15af55b 100644 --- a/test/integration/docker/deployer/app_with_roles/config/deploy.yml +++ b/test/integration/docker/deployer/app_with_roles/config/deploy.yml @@ -9,6 +9,11 @@ servers: hosts: - vm3 cmd: sleep infinity +proxy: + enabled: true + hosts: + - vm2 + deploy_timeout: 2s asset_path: /usr/share/nginx/html/versions diff --git a/test/integration/integration_test.rb b/test/integration/integration_test.rb index ea445d9e..fd23e579 100644 --- a/test/integration/integration_test.rb +++ b/test/integration/integration_test.rb @@ -31,7 +31,7 @@ class IntegrationTest < ActiveSupport::TestCase succeeded = system("cd test/integration && #{command}") end - raise "Command `#{command}` failed with error code `#{$?}`" if !succeeded && raise_on_error + raise "Command `#{command}` failed with error code `#{$?}`, and output:\n#{result}" if !succeeded && raise_on_error result end diff --git a/test/integration/proxy_test.rb b/test/integration/proxy_test.rb new file mode 100644 index 00000000..1f444266 --- /dev/null +++ b/test/integration/proxy_test.rb @@ -0,0 +1,84 @@ +require_relative "integration_test" + +class ProxyTest < IntegrationTest + setup do + @app = "app_with_roles" + end + + test "boot, reboot, stop, start, restart, logs, remove" do + kamal :envify + + kamal :proxy, :boot + assert_proxy_running + + output = kamal :proxy, :reboot, "-y", "--verbose", capture: true + assert_proxy_running + assert_hooks_ran "pre-traefik-reboot", "post-traefik-reboot" + assert_match /Rebooting Traefik on vm1,vm2.../, output + assert_match /Rebooted Traefik on vm1,vm2/, output + + output = kamal :proxy, :reboot, "--rolling", "-y", "--verbose", capture: true + assert_proxy_running + assert_hooks_ran "pre-traefik-reboot", "post-traefik-reboot" + assert_match /Rebooting Traefik on vm1.../, output + assert_match /Rebooted Traefik on vm1/, output + assert_match /Rebooting Traefik on vm2.../, output + assert_match /Rebooted Traefik on vm2/, output + + kamal :proxy, :boot + assert_proxy_running + assert_traefik_running + + # Check booting when booted doesn't raise an error + kamal :proxy, :stop + assert_proxy_not_running + assert_traefik_not_running + + # Check booting when stopped works + kamal :proxy, :boot + assert_proxy_running + assert_traefik_running + + kamal :proxy, :stop + assert_proxy_not_running + assert_traefik_not_running + + kamal :proxy, :start + assert_proxy_running + assert_traefik_running + + kamal :proxy, :restart + assert_proxy_running + assert_traefik_running + + logs = kamal :proxy, :logs, capture: true + assert_match /Traefik version [\d.]+ built on/, logs + + kamal :proxy, :remove + assert_proxy_not_running + assert_traefik_not_running + + kamal :env, :delete + end + + private + def assert_proxy_running + assert_match /basecamp\/kamal-proxy:latest \"kamal-proxy run\"/, proxy_details + end + + def assert_proxy_not_running + assert_no_match /basecamp\/kamal-proxy:latest \"kamal-proxy run\"/, proxy_details + end + + def assert_traefik_running + assert_match /traefik:v2.10 "\/entrypoint.sh/, proxy_details + end + + def assert_traefik_not_running + assert_no_match /traefik:v2.10 "\/entrypoint.sh/, proxy_details + end + + def proxy_details + kamal :proxy, :details, capture: true + end +end