From b130bc0321e2ed9edc72d5d5d05eec17111fb036 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Thu, 6 Feb 2025 14:45:37 +0000 Subject: [PATCH 01/13] Fix the docker build Somewhere along the way the docker build broke, it now needs yaml-dev to be installed. Maybe the underlying image changed? Update to 3.4-alpine while we are here. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d8346804..95c97570 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ruby:3.3-alpine +FROM ruby:3.4-alpine # Install docker/buildx-bin COPY --from=docker/buildx-bin /buildx /usr/libexec/docker/cli-plugins/docker-buildx @@ -13,7 +13,7 @@ COPY Gemfile Gemfile.lock kamal.gemspec ./ COPY lib/kamal/version.rb /kamal/lib/kamal/version.rb # Install system dependencies -RUN apk add --no-cache build-base git docker openrc openssh-client-default \ +RUN apk add --no-cache build-base git docker openrc openssh-client-default yaml-dev \ && rc-update add docker boot \ && gem install bundler --version=2.4.3 \ && bundle install From 6f29d4e78bc29c3392f54f93ea0451ad1ff68b13 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 7 Feb 2025 15:27:30 +0000 Subject: [PATCH 02/13] Bump version for 2.5.2 --- Gemfile.lock | 2 +- lib/kamal/version.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 56ba684b..d129c3f3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - kamal (2.5.1) + kamal (2.5.2) activesupport (>= 7.0) base64 (~> 0.2) bcrypt_pbkdf (~> 1.0) diff --git a/lib/kamal/version.rb b/lib/kamal/version.rb index eefac514..57693c73 100644 --- a/lib/kamal/version.rb +++ b/lib/kamal/version.rb @@ -1,3 +1,3 @@ module Kamal - VERSION = "2.5.1" + VERSION = "2.5.2" end From 32ab72089a227212910031776e784a08f58a8b93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Feb 2025 19:26:36 +0000 Subject: [PATCH 03/13] Bump rack from 3.1.8 to 3.1.10 in the bundler group across 1 directory Bumps the bundler group with 1 update in the / directory: [rack](https://github.com/rack/rack). Updates `rack` from 3.1.8 to 3.1.10 - [Release notes](https://github.com/rack/rack/releases) - [Changelog](https://github.com/rack/rack/blob/main/CHANGELOG.md) - [Commits](https://github.com/rack/rack/compare/v3.1.8...v3.1.10) --- updated-dependencies: - dependency-name: rack dependency-type: indirect dependency-group: bundler ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index d129c3f3..96bf259c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -99,7 +99,7 @@ GEM date stringio racc (1.8.1) - rack (3.1.8) + rack (3.1.10) rack-session (2.0.0) rack (>= 3.0.0) rack-test (2.1.0) From f01238112e8d5adaa83a3ee631db425a98ec819a Mon Sep 17 00:00:00 2001 From: Michael Smart Date: Fri, 21 Feb 2025 11:23:33 +0100 Subject: [PATCH 04/13] Update nokogiri to 1.18.3 See: https://github.com/sparklemotion/nokogiri/releases/tag/v1.18.0 - required for ruby 3.4 compatibility - add more platforms to lockfile to support docker build process, due to changes in the nokogiri native gem setup where -musl and -gnu linux platforms are no longer interchangeable - bundler >= 2.5.6 required according to Nokogiri release notes, so updated to current latest version (2.6.5) --- Dockerfile | 2 +- Gemfile.lock | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 95c97570..e8b4e4ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY lib/kamal/version.rb /kamal/lib/kamal/version.rb # Install system dependencies RUN apk add --no-cache build-base git docker openrc openssh-client-default yaml-dev \ && rc-update add docker boot \ - && gem install bundler --version=2.4.3 \ + && gem install bundler --version=2.6.5 \ && bundle install # Copy the rest of our application code into the container. diff --git a/Gemfile.lock b/Gemfile.lock index d129c3f3..f47309aa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -48,8 +48,6 @@ GEM ast (2.4.2) base64 (0.2.0) bcrypt_pbkdf (1.1.1) - bcrypt_pbkdf (1.1.1-arm64-darwin) - bcrypt_pbkdf (1.1.1-x86_64-darwin) benchmark (0.4.0) bigdecimal (3.1.8) builder (3.3.0) @@ -84,11 +82,15 @@ GEM net-sftp (4.0.0) net-ssh (>= 5.0.0, < 8.0.0) net-ssh (7.3.0) - nokogiri (1.17.2-arm64-darwin) + nokogiri (1.18.3-aarch64-linux-musl) racc (~> 1.4) - nokogiri (1.17.2-x86_64-darwin) + nokogiri (1.18.3-arm64-darwin) racc (~> 1.4) - nokogiri (1.17.2-x86_64-linux) + nokogiri (1.18.3-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.18.3-x86_64-linux-gnu) + racc (~> 1.4) + nokogiri (1.18.3-x86_64-linux-musl) racc (~> 1.4) ostruct (0.6.1) parallel (1.26.3) @@ -177,9 +179,11 @@ GEM zeitwerk (2.7.1) PLATFORMS + aarch64-linux-musl arm64-darwin x86_64-darwin x86_64-linux + x86_64-linux-musl DEPENDENCIES debug @@ -189,4 +193,4 @@ DEPENDENCIES rubocop-rails-omakase BUNDLED WITH - 2.4.3 + 2.6.5 From 62dfa45ee6f71729a540f5fb64c8f4b8771e0861 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Thu, 27 Feb 2025 10:27:27 +0000 Subject: [PATCH 05/13] Bump version for 2.5.3 --- Gemfile.lock | 2 +- lib/kamal/version.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index f47309aa..70849141 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - kamal (2.5.2) + kamal (2.5.3) activesupport (>= 7.0) base64 (~> 0.2) bcrypt_pbkdf (~> 1.0) diff --git a/lib/kamal/version.rb b/lib/kamal/version.rb index 57693c73..3099a7cc 100644 --- a/lib/kamal/version.rb +++ b/lib/kamal/version.rb @@ -1,3 +1,3 @@ module Kamal - VERSION = "2.5.2" + VERSION = "2.5.3" end From 973fa1a7ff48e452c0a7a3d31026b23879069f0a Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Sun, 2 Mar 2025 09:27:50 -0700 Subject: [PATCH 06/13] Adds the ability to alias/map secrets --- lib/kamal/configuration/docs/env.yml | 24 ++++++++++++++++++++++++ lib/kamal/configuration/env.rb | 10 +++++++++- test/configuration/env_test.rb | 14 ++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/lib/kamal/configuration/docs/env.yml b/lib/kamal/configuration/docs/env.yml index 98521ae4..23b5b60b 100644 --- a/lib/kamal/configuration/docs/env.yml +++ b/lib/kamal/configuration/docs/env.yml @@ -51,6 +51,30 @@ env: secret: - DB_PASSWORD +# Aliased secrets +# +# You can also alias secrets to other secrets using a `:` separator. +# +# This is useful when the ENV name is different from the secret name. For example, if you have two +# places where you need to define the ENV variable `DB_PASSWORD`, but the value is different depending +# on the context. +# +# ```shell +# SECRETS=$(kamal secrets fetch ...) +# +# MAIN_DB_PASSWORD=$(kamal secrets extract MAIN_DB_PASSWORD $SECRETS) +# SECONDARY_DB_PASSWORD=$(kamal secrets extract SECONDARY_DB_PASSWORD $SECRETS) +# ``` +accessories: + main_db_accessory: + env: + secret: + - DB_PASSWORD:MAIN_DB_PASSWORD + secondary_db_accessory: + env: + secret: + - DB_PASSWORD:SECONDARY_DB_PASSWORD + # Tags # # Tags are used to add extra env variables to specific hosts. diff --git a/lib/kamal/configuration/env.rb b/lib/kamal/configuration/env.rb index 8e52d9e4..077f1cf2 100644 --- a/lib/kamal/configuration/env.rb +++ b/lib/kamal/configuration/env.rb @@ -18,7 +18,7 @@ class Kamal::Configuration::Env end def secrets_io - Kamal::EnvFile.new(secret_keys.to_h { |key| [ key, secrets[key] ] }).to_io + Kamal::EnvFile.new(secrets_hash).to_io end def merge(other) @@ -26,4 +26,12 @@ class Kamal::Configuration::Env config: { "clear" => clear.merge(other.clear), "secret" => secret_keys | other.secret_keys }, secrets: secrets end + + private + def secrets_hash + secret_keys.to_h do |key| + key_name, key_aliased_to = key.split(":") + [ key_name, secrets[key_aliased_to || key_name] ] + end + end end diff --git a/test/configuration/env_test.rb b/test/configuration/env_test.rb index 627d3a6c..29cf0d85 100644 --- a/test/configuration/env_test.rb +++ b/test/configuration/env_test.rb @@ -48,6 +48,20 @@ class ConfigurationEnvTest < ActiveSupport::TestCase end end + test "aliased secrets" do + with_test_secrets("secrets" => "ALIASED_PASSWORD=hello") do + config = { + "secret" => [ "PASSWORD:ALIASED_PASSWORD" ], + "clear" => {} + } + + assert_config \ + config: config, + clear: {}, + secrets: { "PASSWORD" => "hello" } + end + end + private def assert_config(config:, clear: {}, secrets: {}) env = Kamal::Configuration::Env.new config: config, secrets: Kamal::Secrets.new From 36f4e90a766d48d38df267fdad5015d029187faf Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 7 Mar 2025 11:33:17 +0000 Subject: [PATCH 07/13] Update to kamal-proxy 0.8.6 Includes a fix for locking the proxy while draining --- lib/kamal/configuration.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index 023a2d7c..065a65b0 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -14,7 +14,7 @@ class Kamal::Configuration include Validation - PROXY_MINIMUM_VERSION = "v0.8.4" + PROXY_MINIMUM_VERSION = "v0.8.6" PROXY_HTTP_PORT = 80 PROXY_HTTPS_PORT = 443 PROXY_LOG_MAX_SIZE = "10m" From 8098ed1fd11a4c3c7b89b415db9aafa52b556e18 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Mar 2025 22:29:34 +0000 Subject: [PATCH 08/13] Bump the bundler group across 1 directory with 2 updates Bumps the bundler group with 2 updates in the / directory: [rack](https://github.com/rack/rack) and [uri](https://github.com/ruby/uri). Updates `rack` from 3.1.10 to 3.1.12 - [Release notes](https://github.com/rack/rack/releases) - [Changelog](https://github.com/rack/rack/blob/main/CHANGELOG.md) - [Commits](https://github.com/rack/rack/compare/v3.1.10...v3.1.12) Updates `uri` from 1.0.2 to 1.0.3 - [Release notes](https://github.com/ruby/uri/releases) - [Commits](https://github.com/ruby/uri/compare/v1.0.2...v1.0.3) --- updated-dependencies: - dependency-name: rack dependency-type: indirect dependency-group: bundler - dependency-name: uri dependency-type: indirect dependency-group: bundler ... Signed-off-by: dependabot[bot] --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2f2ce40c..5875f349 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -101,7 +101,7 @@ GEM date stringio racc (1.8.1) - rack (3.1.10) + rack (3.1.12) rack-session (2.0.0) rack (>= 3.0.0) rack-test (2.1.0) @@ -174,7 +174,7 @@ GEM unicode-display_width (3.1.2) unicode-emoji (~> 4.0, >= 4.0.4) unicode-emoji (4.0.4) - uri (1.0.2) + uri (1.0.3) useragent (0.16.11) zeitwerk (2.7.1) From 3c9a3f226427c196a447ba9e77755ae9f78e1a3b Mon Sep 17 00:00:00 2001 From: Kevin McConnell Date: Wed, 12 Mar 2025 10:50:16 +0000 Subject: [PATCH 09/13] Require proxy version v0.8.7 --- lib/kamal/configuration.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index 065a65b0..1c0b79cf 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -14,7 +14,7 @@ class Kamal::Configuration include Validation - PROXY_MINIMUM_VERSION = "v0.8.6" + PROXY_MINIMUM_VERSION = "v0.8.7" PROXY_HTTP_PORT = 80 PROXY_HTTPS_PORT = 443 PROXY_LOG_MAX_SIZE = "10m" From d4ab010b0149190a4f9b57d08fe875d1b960b0e6 Mon Sep 17 00:00:00 2001 From: Kevin McConnell Date: Wed, 12 Mar 2025 10:49:53 +0000 Subject: [PATCH 10/13] Add `ssl_redirect` proxy option --- lib/kamal/configuration/docs/proxy.yml | 7 +++++++ lib/kamal/configuration/proxy.rb | 1 + 2 files changed, 8 insertions(+) diff --git a/lib/kamal/configuration/docs/proxy.yml b/lib/kamal/configuration/docs/proxy.yml index 9ed6a97f..49c11ac8 100644 --- a/lib/kamal/configuration/docs/proxy.yml +++ b/lib/kamal/configuration/docs/proxy.yml @@ -52,6 +52,13 @@ proxy: # Defaults to `false`: ssl: true + # SSL redirect + # + # By default, kamal-proxy will redirect all HTTP requests to HTTPS when SSL is enabled. + # If you prefer that HTTP traffic is passed through to your application (along with + # HTTPS traffic), you can disable this redirect by setting `ssl_redirect: false`: + ssl_redirect: false + # Forward headers # # Whether to forward the `X-Forwarded-For` and `X-Forwarded-Proto` headers. diff --git a/lib/kamal/configuration/proxy.rb b/lib/kamal/configuration/proxy.rb index 6232c3e0..a8272e88 100644 --- a/lib/kamal/configuration/proxy.rb +++ b/lib/kamal/configuration/proxy.rb @@ -42,6 +42,7 @@ class Kamal::Configuration::Proxy "max-request-body": proxy_config.dig("buffering", "max_request_body"), "max-response-body": proxy_config.dig("buffering", "max_response_body"), "forward-headers": proxy_config.dig("forward_headers"), + "tls-redirect": proxy_config.dig("ssl_redirect"), "log-request-header": proxy_config.dig("logging", "request_headers") || DEFAULT_LOG_REQUEST_HEADERS, "log-response-header": proxy_config.dig("logging", "response_headers") }.compact From 7fa27faaca90cbfde673145a8a96d5adb323c988 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Mon, 17 Mar 2025 16:58:15 +0000 Subject: [PATCH 11/13] Use xargs to handle spaces in proxy options We cat the options file, append the proxy image and then pass it to xargs to ensure it handles spaces correctly. Works better than using eval which can handle spaces but tries to evaluate things like backticks. Fixes: https://github.com/basecamp/kamal/issues/1448 --- lib/kamal/commands/proxy.rb | 18 ++++++++--------- test/cli/proxy_test.rb | 14 ++++++------- test/commands/proxy_test.rb | 6 +++--- .../app_with_traefik/.kamal/hooks/pre-deploy | 3 ++- test/integration/proxy_test.rb | 20 +++++++++++++++++++ 5 files changed, 41 insertions(+), 20 deletions(-) diff --git a/lib/kamal/commands/proxy.rb b/lib/kamal/commands/proxy.rb index acff3dbd..6ca87e02 100644 --- a/lib/kamal/commands/proxy.rb +++ b/lib/kamal/commands/proxy.rb @@ -2,14 +2,14 @@ class Kamal::Commands::Proxy < Kamal::Commands::Base delegate :argumentize, :optionize, to: Kamal::Utils def run - docker :run, - "--name", container_name, - "--network", "kamal", - "--detach", - "--restart", "unless-stopped", - "--volume", "kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", - "\$\(#{get_boot_options.join(" ")}\)", - config.proxy_image + pipe \ + [ :echo, "\$\(#{get_boot_options.join(" ")}\) #{config.proxy_image}" ], + xargs(docker(:run, + "--name", container_name, + "--network", "kamal", + "--detach", + "--restart", "unless-stopped", + "--volume", "kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy")) end def start @@ -73,7 +73,7 @@ class Kamal::Commands::Proxy < Kamal::Commands::Base end def get_boot_options - combine [ :cat, config.proxy_options_file ], [ :echo, "\"#{config.proxy_options_default.join(" ")}\"" ], by: "||" + combine [ :cat, config.proxy_options_file, "2>", "/dev/null" ], [ :echo, "\"#{config.proxy_options_default.join(" ")}\"" ], by: "||" end def reset_boot_options diff --git a/test/cli/proxy_test.rb b/test/cli/proxy_test.rb index b9de3e15..4880a839 100644 --- a/test/cli/proxy_test.rb +++ b/test/cli/proxy_test.rb @@ -4,7 +4,7 @@ class CliProxyTest < CliTestCase test "boot" do run_command("boot").tap do |output| assert_match "docker login", output - assert_match "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image}", output + assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output end end @@ -18,7 +18,7 @@ class CliProxyTest < CliTestCase exception = assert_raises do run_command("boot").tap do |output| assert_match "docker login", output - assert_match "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image}", output + assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output end end @@ -36,7 +36,7 @@ class CliProxyTest < CliTestCase run_command("boot").tap do |output| assert_match "docker login", output - assert_match "docker container start kamal-proxy || docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image}", output + assert_match "docker container start kamal-proxy || echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output end ensure Thread.report_on_exception = false @@ -56,12 +56,12 @@ class CliProxyTest < CliTestCase run_command("reboot", "-y").tap do |output| assert_match "docker container stop kamal-proxy on 1.1.1.1", output assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.1", output - assert_match "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image} on 1.1.1.1", output + assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy on 1.1.1.1", output assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target=\"abcdefabcdef:80\" --deploy-timeout=\"6s\" --drain-timeout=\"30s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\" on 1.1.1.1", output assert_match "docker container stop kamal-proxy on 1.1.1.2", output assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.2", output - assert_match "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image} on 1.1.1.2", output + assert_match "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy on 1.1.1.2", output assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target=\"abcdefabcdef:80\" --deploy-timeout=\"6s\" --drain-timeout=\"30s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\" on 1.1.1.2", output end end @@ -196,7 +196,7 @@ class CliProxyTest < CliTestCase assert_match "/usr/bin/env mkdir -p .kamal", output assert_match "docker network create kamal", output assert_match "docker login -u [REDACTED] -p [REDACTED]", output - assert_match "docker container start kamal-proxy || docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION}", output + assert_match "docker container start kamal-proxy || echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", output assert_match "/usr/bin/env mkdir -p .kamal", output assert_match %r{docker rename app-web-latest app-web-latest_replaced_.*}, output assert_match "/usr/bin/env mkdir -p .kamal/apps/app/env/roles", output @@ -316,7 +316,7 @@ class CliProxyTest < CliTestCase test "boot_config get" do SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:cat, ".kamal/proxy/options", "||", :echo, "\"--publish 80:80 --publish 443:443 --log-opt max-size=10m\"") + .with(:cat, ".kamal/proxy/options", "2>", "/dev/null", "||", :echo, "\"--publish 80:80 --publish 443:443 --log-opt max-size=10m\"") .returns("--publish 80:80 --publish 8443:443 --label=foo=bar") .twice diff --git a/test/commands/proxy_test.rb b/test/commands/proxy_test.rb index b7cc9f3d..e42415b1 100644 --- a/test/commands/proxy_test.rb +++ b/test/commands/proxy_test.rb @@ -15,7 +15,7 @@ class CommandsProxyTest < ActiveSupport::TestCase test "run" do assert_equal \ - "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION}", + "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", new_command.run.join(" ") end @@ -23,7 +23,7 @@ class CommandsProxyTest < ActiveSupport::TestCase @config.delete(:proxy) assert_equal \ - "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION}", + "echo $(cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") basecamp/kamal-proxy:#{Kamal::Configuration::PROXY_MINIMUM_VERSION} | xargs docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy", new_command.run.join(" ") end @@ -113,7 +113,7 @@ class CommandsProxyTest < ActiveSupport::TestCase test "get_boot_options" do assert_equal \ - "cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\"", + "cat .kamal/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\"", new_command.get_boot_options.join(" ") end diff --git a/test/integration/docker/deployer/app_with_traefik/.kamal/hooks/pre-deploy b/test/integration/docker/deployer/app_with_traefik/.kamal/hooks/pre-deploy index d0483a43..b2d8b112 100755 --- a/test/integration/docker/deployer/app_with_traefik/.kamal/hooks/pre-deploy +++ b/test/integration/docker/deployer/app_with_traefik/.kamal/hooks/pre-deploy @@ -1,3 +1,4 @@ kamal proxy boot_config set --publish false \ --docker_options label=traefik.http.services.kamal_proxy.loadbalancer.server.scheme=http \ - label=traefik.http.routers.kamal_proxy.rule=PathPrefix\(\`/\`\) + label=traefik.http.routers.kamal_proxy.rule=PathPrefix\(\`/\`\) \ + sysctl=net.ipv4.ip_local_port_range=\"10000\ 60999\" diff --git a/test/integration/proxy_test.rb b/test/integration/proxy_test.rb index d861a887..39cacb94 100644 --- a/test/integration/proxy_test.rb +++ b/test/integration/proxy_test.rb @@ -46,7 +46,27 @@ class ProxyTest < IntegrationTest logs = kamal :proxy, :logs, capture: true assert_match /No previous state to restore/, logs + kamal :proxy, :boot_config, :set, "--docker-options='sysctl net.ipv4.ip_local_port_range=\"10000 60999\"'" + assert_docker_options_in_file + + kamal :proxy, :reboot, "-y" + assert_docker_options_in_container + + kamal :proxy, :boot_config, :reset + kamal :proxy, :remove assert_proxy_not_running end + + private + def assert_docker_options_in_file + boot_config = kamal :proxy, :boot_config, :get, capture: true + assert_match "Host vm1: --publish 80:80 --publish 443:443 --log-opt max-size=10m --sysctl net.ipv4.ip_local_port_range=\"10000 60999\"", boot_config + end + + def assert_docker_options_in_container + assert_equal \ + "{\"net.ipv4.ip_local_port_range\":\"10000 60999\"}", + docker_compose("exec vm1 docker inspect --format '{{ json .HostConfig.Sysctls }}' kamal-proxy", capture: true).strip + end end From c1d8ce7f705bab77967233005011dedef9ecc9e9 Mon Sep 17 00:00:00 2001 From: Camillo Visini Date: Sat, 22 Mar 2025 12:07:22 +0100 Subject: [PATCH 12/13] Add ability to alias secrets for tags Aliasing for secrets was introduced in #1439, but only supported "top-level" secrets. This adds support for aliasing/mapping secrets for tags. --- lib/kamal/configuration/docs/env.yml | 7 +++++++ lib/kamal/configuration/env.rb | 24 +++++++++++++++++------- test/configuration/env/tags_test.rb | 18 ++++++++++++++++++ 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/lib/kamal/configuration/docs/env.yml b/lib/kamal/configuration/docs/env.yml index 23b5b60b..b73e6309 100644 --- a/lib/kamal/configuration/docs/env.yml +++ b/lib/kamal/configuration/docs/env.yml @@ -65,6 +65,13 @@ env: # MAIN_DB_PASSWORD=$(kamal secrets extract MAIN_DB_PASSWORD $SECRETS) # SECONDARY_DB_PASSWORD=$(kamal secrets extract SECONDARY_DB_PASSWORD $SECRETS) # ``` +env: + secret: + - DB_PASSWORD:MAIN_DB_PASSWORD + tags: + secondary_db: + secret: + - DB_PASSWORD:SECONDARY_DB_PASSWORD accessories: main_db_accessory: env: diff --git a/lib/kamal/configuration/env.rb b/lib/kamal/configuration/env.rb index 077f1cf2..cb02caab 100644 --- a/lib/kamal/configuration/env.rb +++ b/lib/kamal/configuration/env.rb @@ -1,7 +1,7 @@ class Kamal::Configuration::Env include Kamal::Configuration::Validation - attr_reader :context, :secrets + attr_reader :context attr_reader :clear, :secret_keys delegate :argumentize, to: Kamal::Utils @@ -11,27 +11,37 @@ class Kamal::Configuration::Env @secret_keys = config.fetch("secret", []) @context = context validate! config, context: context, with: Kamal::Configuration::Validator::Env + @secret_map = build_secret_map(@secret_keys) end def clear_args argumentize("--env", clear) end + def secrets + @resolved_secrets ||= resolve_secrets + end + def secrets_io - Kamal::EnvFile.new(secrets_hash).to_io + Kamal::EnvFile.new(secrets).to_io end def merge(other) self.class.new \ config: { "clear" => clear.merge(other.clear), "secret" => secret_keys | other.secret_keys }, - secrets: secrets + secrets: @secrets end private - def secrets_hash - secret_keys.to_h do |key| - key_name, key_aliased_to = key.split(":") - [ key_name, secrets[key_aliased_to || key_name] ] + def build_secret_map(secret_keys) + Array(secret_keys).to_h do |key| + key_name, key_aliased_to = key.split(":", 2) + key_aliased_to ||= key_name + [ key_name, key_aliased_to ] end end + + def resolve_secrets + @secret_map.transform_values { |secret_key| @secrets[secret_key] } + end end diff --git a/test/configuration/env/tags_test.rb b/test/configuration/env/tags_test.rb index 7db61777..577ff090 100644 --- a/test/configuration/env/tags_test.rb +++ b/test/configuration/env/tags_test.rb @@ -96,6 +96,24 @@ class ConfigurationEnvTagsTest < ActiveSupport::TestCase end end + test "aliased tag secret env" do + with_test_secrets("secrets" => "PASSWORD=hello\nALIASED_PASSWORD=aliased_hello") do + deploy = { + service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, + servers: [ { "1.1.1.1" => "secrets" } ], + builder: { "arch" => "amd64" }, + env: { + "tags" => { + "secrets" => { "secret" => [ "PASSWORD:ALIASED_PASSWORD" ] } + } + } + } + + config = Kamal::Configuration.new(deploy) + assert_equal "aliased_hello", config.role("web").env("1.1.1.1").secrets["PASSWORD"] + end + end + test "tag clear env" do deploy = { service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, From fb95b38e73a8068d1b5142ad5917c73c99a5f7a7 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Mon, 24 Mar 2025 09:20:45 +0000 Subject: [PATCH 13/13] Tidy up the env secrets handling The secrets accessor was only used in the tests so remove it. Skip the memoization, it makes things slightly harder to follow and it's not needed. --- lib/kamal/configuration/env.rb | 23 +++++++---------------- test/configuration/env/tags_test.rb | 4 ++-- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/lib/kamal/configuration/env.rb b/lib/kamal/configuration/env.rb index cb02caab..bebcf1db 100644 --- a/lib/kamal/configuration/env.rb +++ b/lib/kamal/configuration/env.rb @@ -1,8 +1,7 @@ class Kamal::Configuration::Env include Kamal::Configuration::Validation - attr_reader :context - attr_reader :clear, :secret_keys + attr_reader :context, :clear, :secret_keys delegate :argumentize, to: Kamal::Utils def initialize(config:, secrets:, context: "env") @@ -11,19 +10,14 @@ class Kamal::Configuration::Env @secret_keys = config.fetch("secret", []) @context = context validate! config, context: context, with: Kamal::Configuration::Validator::Env - @secret_map = build_secret_map(@secret_keys) end def clear_args argumentize("--env", clear) end - def secrets - @resolved_secrets ||= resolve_secrets - end - def secrets_io - Kamal::EnvFile.new(secrets).to_io + Kamal::EnvFile.new(aliased_secrets).to_io end def merge(other) @@ -33,15 +27,12 @@ class Kamal::Configuration::Env end private - def build_secret_map(secret_keys) - Array(secret_keys).to_h do |key| - key_name, key_aliased_to = key.split(":", 2) - key_aliased_to ||= key_name - [ key_name, key_aliased_to ] - end + def aliased_secrets + secret_keys.to_h { |key| extract_alias(key) }.transform_values { |secret_key| @secrets[secret_key] } end - def resolve_secrets - @secret_map.transform_values { |secret_key| @secrets[secret_key] } + def extract_alias(key) + key_name, key_aliased_to = key.split(":", 2) + [ key_name, key_aliased_to || key_name ] end end diff --git a/test/configuration/env/tags_test.rb b/test/configuration/env/tags_test.rb index 577ff090..e857faaa 100644 --- a/test/configuration/env/tags_test.rb +++ b/test/configuration/env/tags_test.rb @@ -92,7 +92,7 @@ class ConfigurationEnvTagsTest < ActiveSupport::TestCase } config = Kamal::Configuration.new(deploy) - assert_equal "hello", config.role("web").env("1.1.1.1").secrets["PASSWORD"] + assert_equal "PASSWORD=hello\n", config.role("web").env("1.1.1.1").secrets_io.string end end @@ -110,7 +110,7 @@ class ConfigurationEnvTagsTest < ActiveSupport::TestCase } config = Kamal::Configuration.new(deploy) - assert_equal "aliased_hello", config.role("web").env("1.1.1.1").secrets["PASSWORD"] + assert_equal "PASSWORD=aliased_hello\n", config.role("web").env("1.1.1.1").secrets_io.string end end