From 3c91a839425b18a3e2d5f9756cc9e3e324596f7d Mon Sep 17 00:00:00 2001 From: Ralf Schmitz Bongiolo Date: Thu, 10 Oct 2024 21:41:09 -0400 Subject: [PATCH 01/26] feat(secrets): add Doppler adapter --- lib/kamal/secrets/adapters/doppler.rb | 28 ++++++++ test/secrets/doppler_adapter_test.rb | 100 ++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 lib/kamal/secrets/adapters/doppler.rb create mode 100644 test/secrets/doppler_adapter_test.rb diff --git a/lib/kamal/secrets/adapters/doppler.rb b/lib/kamal/secrets/adapters/doppler.rb new file mode 100644 index 00000000..caa1833e --- /dev/null +++ b/lib/kamal/secrets/adapters/doppler.rb @@ -0,0 +1,28 @@ +class Kamal::Secrets::Adapters::Doppler < Kamal::Secrets::Adapters::Base + private + def login(account) + unless loggedin?(account) + `doppler login -y` + raise RuntimeError, "Failed to login to Doppler" unless $?.success? + end + end + + def loggedin?(account) + `doppler me --json 2> /dev/null` + $?.success? + end + + def fetch_secrets(secrets, account:, session:) + project, config = account.split("/") + + raise RuntimeError, "Missing project or config from --acount=project/config option" unless project && config + raise RuntimeError, "Using --from option or FOLDER/SECRET is not supported by Doppler" if secrets.any?(/\//) + + items = `doppler secrets get #{secrets.map(&:shellescape).join(" ")} --json -p #{project} -c #{config}` + raise RuntimeError, "Could not read #{secrets} from Doppler" unless $?.success? + + items = JSON.parse(items) + + items.transform_values { |value| value["computed"] } + end +end diff --git a/test/secrets/doppler_adapter_test.rb b/test/secrets/doppler_adapter_test.rb new file mode 100644 index 00000000..c7cda494 --- /dev/null +++ b/test/secrets/doppler_adapter_test.rb @@ -0,0 +1,100 @@ +require "test_helper" + +class DopplerAdapterTest < SecretAdapterTestCase + setup do + `true` # Ensure $? is 0 + end + + test "fetch" do + stub_ticks.with("doppler me --json 2> /dev/null") + + stub_ticks + .with("doppler secrets get SECRET1 FSECRET1 FSECRET2 --json -p my-project -c prd") + .returns(<<~JSON) + { + "SECRET1": { + "computed":"secret1", + "computedVisibility":"unmasked", + "note":"" + }, + "FSECRET1": { + "computed":"fsecret1", + "computedVisibility":"unmasked", + "note":"" + }, + "FSECRET2": { + "computed":"fsecret2", + "computedVisibility":"unmasked", + "note":"" + } + } + JSON + + json = JSON.parse(shellunescape(run_command("fetch", "SECRET1", "FSECRET1", "FSECRET2"))) + + expected_json = { + "SECRET1"=>"secret1", + "FSECRET1"=>"fsecret1", + "FSECRET2"=>"fsecret2" + } + + assert_equal expected_json, json + end + + test "fetch with from" do + stub_ticks.with("doppler me --json 2> /dev/null") + + error = assert_raises RuntimeError do + run_command("fetch", "--from", "FOLDER1", "FSECRET1", "FSECRET2") + end + + assert_match(/Using --from option or FOLDER\/SECRET is not supported by Doppler/, error.message) + end + + test "fetch with folder in secret" do + stub_ticks.with("doppler me --json 2> /dev/null") + + error = assert_raises RuntimeError do + run_command("fetch", "FOLDER1/FSECRET1", "SECRET2") + end + + assert_match(/Using --from option or FOLDER\/SECRET is not supported by Doppler/, error.message) + end + + test "fetch with signin" do + stub_ticks_with("doppler me --json 2> /dev/null", succeed: false) + stub_ticks_with("doppler login -y", succeed: true).returns("") + stub_ticks.with("doppler secrets get SECRET1 --json -p my-project -c prd").returns(single_item_json) + + json = JSON.parse(shellunescape(run_command("fetch", "SECRET1"))) + + expected_json = { + "SECRET1"=>"secret1" + } + + assert_equal expected_json, json + end + + private + def run_command(*command) + stdouted do + Kamal::Cli::Secrets.start \ + [ *command, + "-c", "test/fixtures/deploy_with_accessories.yml", + "--adapter", "doppler", + "--account", "my-project/prd" ] + end + end + + def single_item_json + <<~JSON + { + "SECRET1": { + "computed":"secret1", + "computedVisibility":"unmasked", + "note":"" + } + } + JSON + end +end From b4df51b8b407eb3d6a84e52ba68cb73a6e61deda Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Sat, 12 Oct 2024 21:27:56 +0400 Subject: [PATCH 02/26] Added example how to read the Ruby version from the .ruby-version file. --- lib/kamal/cli/templates/deploy.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/kamal/cli/templates/deploy.yml b/lib/kamal/cli/templates/deploy.yml index cff7f062..5a0d2cf1 100644 --- a/lib/kamal/cli/templates/deploy.yml +++ b/lib/kamal/cli/templates/deploy.yml @@ -16,8 +16,8 @@ servers: # Enable SSL auto certification via Let's Encrypt and allow for multiple apps on a single web server. # Remove this section when using multiple web servers and ensure you terminate SSL at your load balancer. # -# Note: If using Cloudflare, set encryption mode in SSL/TLS setting to "Full" to enable CF-to-app encryption. -proxy: +# Note: If using Cloudflare, set encryption mode in SSL/TLS setting to "Full" to enable CF-to-app encryption. +proxy: ssl: true host: app.example.com # Proxy connects to your container on port 80 by default. @@ -36,6 +36,9 @@ registry: # Configure builder setup. builder: arch: amd64 + # Pass in additional build args needed for your Dockerfile. + # args: + # RUBY_VERSION: <%= File.read('.ruby-version').strip %> # Inject ENV variables into containers (secrets come from .kamal/secrets). # From 6856742eca5ddcdc62a76a20d530bff695fdbc35 Mon Sep 17 00:00:00 2001 From: Justin Dell Date: Mon, 21 Oct 2024 09:19:06 -0500 Subject: [PATCH 03/26] add secrets adapter for aws secrets manager --- .../secrets/adapters/aws_secretsmanager.rb | 25 ++++++ .../aws_secretsmanager_adapter_test.rb | 87 +++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 lib/kamal/secrets/adapters/aws_secretsmanager.rb create mode 100644 test/secrets/aws_secretsmanager_adapter_test.rb diff --git a/lib/kamal/secrets/adapters/aws_secretsmanager.rb b/lib/kamal/secrets/adapters/aws_secretsmanager.rb new file mode 100644 index 00000000..8d6f2ead --- /dev/null +++ b/lib/kamal/secrets/adapters/aws_secretsmanager.rb @@ -0,0 +1,25 @@ +class Kamal::Secrets::Adapters::AwsSecretsmanager < Kamal::Secrets::Adapters::Base + private + def login(_account) + nil + end + + def fetch_secrets(secrets, account:, session:) + {}.tap do |results| + JSON.parse(get_from_secrets_manager(secrets, account: account))["SecretValues"].each do |secret| + secret_name = secret["Name"] + secret_string = JSON.parse(secret["SecretString"]) + + secret_string.each do |key, value| + results["#{secret_name}/#{key}"] = value + end + end + end + end + + def get_from_secrets_manager(secrets, account:) + `aws secretsmanager batch-get-secret-value --secret-id-list #{secrets.map(&:shellescape).join(" ")} --profile #{account}`.tap do + raise RuntimeError, "Could not read #{secret} from AWS Secrets Manager" unless $?.success? + end + end +end diff --git a/test/secrets/aws_secretsmanager_adapter_test.rb b/test/secrets/aws_secretsmanager_adapter_test.rb new file mode 100644 index 00000000..26952034 --- /dev/null +++ b/test/secrets/aws_secretsmanager_adapter_test.rb @@ -0,0 +1,87 @@ +require "test_helper" + +class AwsSecretsmanagerAdapterTest < SecretAdapterTestCase + test "fetch" do + stub_ticks + .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 secret2/KEY3 --profile default") + .returns(<<~JSON) + { + "SecretValues": [ + { + "ARN": "arn:aws:secretsmanager:us-east-1:aaaaaaaaaaaa:secret:secret", + "Name": "secret", + "VersionId": "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv", + "SecretString": "{\\"KEY1\\":\\"VALUE1\\", \\"KEY2\\":\\"VALUE2\\"}", + "VersionStages": [ + "AWSCURRENT" + ], + "CreatedDate": "2024-01-01T00:00:00.000000" + }, + { + "ARN": "arn:aws:secretsmanager:us-east-1:aaaaaaaaaaaa:secret:secret2", + "Name": "secret2", + "VersionId": "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv", + "SecretString": "{\\"KEY3\\":\\"VALUE3\\"}", + "VersionStages": [ + "AWSCURRENT" + ], + "CreatedDate": "2024-01-01T00:00:00.000000" + } + ], + "Errors": [] + } + JSON + + json = JSON.parse(shellunescape(run_command("fetch", "secret/KEY1", "secret/KEY2", "secret2/KEY3"))) + + expected_json = { + "secret/KEY1"=>"VALUE1", + "secret/KEY2"=>"VALUE2", + "secret2/KEY3"=>"VALUE3" + } + + assert_equal expected_json, json + end + + test "fetch with secret names" do + stub_ticks + .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 --profile default") + .returns(<<~JSON) + { + "SecretValues": [ + { + "ARN": "arn:aws:secretsmanager:us-east-1:aaaaaaaaaaaa:secret:secret", + "Name": "secret", + "VersionId": "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv", + "SecretString": "{\\"KEY1\\":\\"VALUE1\\", \\"KEY2\\":\\"VALUE2\\"}", + "VersionStages": [ + "AWSCURRENT" + ], + "CreatedDate": "2024-01-01T00:00:00.000000" + } + ], + "Errors": [] + } + JSON + + json = JSON.parse(shellunescape(run_command("fetch", "--from", "secret", "KEY1", "KEY2"))) + + expected_json = { + "secret/KEY1"=>"VALUE1", + "secret/KEY2"=>"VALUE2" + } + + assert_equal expected_json, json + end + + private + def run_command(*command) + stdouted do + Kamal::Cli::Secrets.start \ + [ *command, + "-c", "test/fixtures/deploy_with_accessories.yml", + "--adapter", "aws_secretsmanager", + "--account", "default" ] + end + end +end From 7e8a8eb6e5965d35991796723318fa12c9278e9a Mon Sep 17 00:00:00 2001 From: David Stosik <816901+davidstosik@users.noreply.github.com> Date: Sun, 27 Oct 2024 23:37:47 +0900 Subject: [PATCH 04/26] Remove trailing spaces from deploy.yml template Just a minor cleanup, nothing important. `git` highlighted these spaces in red in my commit so I thought I'd remove them. --- lib/kamal/cli/templates/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/kamal/cli/templates/deploy.yml b/lib/kamal/cli/templates/deploy.yml index fc44b6b9..5cc274c4 100644 --- a/lib/kamal/cli/templates/deploy.yml +++ b/lib/kamal/cli/templates/deploy.yml @@ -16,8 +16,8 @@ servers: # Enable SSL auto certification via Let's Encrypt and allow for multiple apps on a single web server. # Remove this section when using multiple web servers and ensure you terminate SSL at your load balancer. # -# Note: If using Cloudflare, set encryption mode in SSL/TLS setting to "Full" to enable CF-to-app encryption. -proxy: +# Note: If using Cloudflare, set encryption mode in SSL/TLS setting to "Full" to enable CF-to-app encryption. +proxy: ssl: true host: app.example.com # Proxy connects to your container on port 80 by default. From c9fff3cb4024f67ff5c4a9c80a06c3675a913bfe Mon Sep 17 00:00:00 2001 From: Justin Dell Date: Mon, 4 Nov 2024 09:14:47 -0600 Subject: [PATCH 05/26] rename secretsmanager to secrets manager --- .../{aws_secretsmanager.rb => aws_secrets_manager.rb} | 2 +- ...er_adapter_test.rb => aws_secrets_manager_adapter_test.rb} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename lib/kamal/secrets/adapters/{aws_secretsmanager.rb => aws_secrets_manager.rb} (92%) rename test/secrets/{aws_secretsmanager_adapter_test.rb => aws_secrets_manager_adapter_test.rb} (96%) diff --git a/lib/kamal/secrets/adapters/aws_secretsmanager.rb b/lib/kamal/secrets/adapters/aws_secrets_manager.rb similarity index 92% rename from lib/kamal/secrets/adapters/aws_secretsmanager.rb rename to lib/kamal/secrets/adapters/aws_secrets_manager.rb index 8d6f2ead..7f834c3e 100644 --- a/lib/kamal/secrets/adapters/aws_secretsmanager.rb +++ b/lib/kamal/secrets/adapters/aws_secrets_manager.rb @@ -1,4 +1,4 @@ -class Kamal::Secrets::Adapters::AwsSecretsmanager < Kamal::Secrets::Adapters::Base +class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Base private def login(_account) nil diff --git a/test/secrets/aws_secretsmanager_adapter_test.rb b/test/secrets/aws_secrets_manager_adapter_test.rb similarity index 96% rename from test/secrets/aws_secretsmanager_adapter_test.rb rename to test/secrets/aws_secrets_manager_adapter_test.rb index 26952034..eb425554 100644 --- a/test/secrets/aws_secretsmanager_adapter_test.rb +++ b/test/secrets/aws_secrets_manager_adapter_test.rb @@ -1,6 +1,6 @@ require "test_helper" -class AwsSecretsmanagerAdapterTest < SecretAdapterTestCase +class AwsSecretsManagerAdapterTest < SecretAdapterTestCase test "fetch" do stub_ticks .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 secret2/KEY3 --profile default") @@ -80,7 +80,7 @@ class AwsSecretsmanagerAdapterTest < SecretAdapterTestCase Kamal::Cli::Secrets.start \ [ *command, "-c", "test/fixtures/deploy_with_accessories.yml", - "--adapter", "aws_secretsmanager", + "--adapter", "aws_secrets_manager", "--account", "default" ] end end From e26694541386acdd13dbb13ea23447b6002d8a29 Mon Sep 17 00:00:00 2001 From: Justin Dell Date: Mon, 4 Nov 2024 09:18:56 -0600 Subject: [PATCH 06/26] implement check_dependencies! --- lib/kamal/secrets/adapters/aws_secrets_manager.rb | 9 +++++++++ test/secrets/aws_secrets_manager_adapter_test.rb | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/lib/kamal/secrets/adapters/aws_secrets_manager.rb b/lib/kamal/secrets/adapters/aws_secrets_manager.rb index 7f834c3e..1da48b94 100644 --- a/lib/kamal/secrets/adapters/aws_secrets_manager.rb +++ b/lib/kamal/secrets/adapters/aws_secrets_manager.rb @@ -22,4 +22,13 @@ class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Ba raise RuntimeError, "Could not read #{secret} from AWS Secrets Manager" unless $?.success? end end + + def check_dependencies! + raise RuntimeError, "AWS CLI is not installed" unless cli_installed? + end + + def cli_installed? + `aws --version 2> /dev/null` + $?.success? + end end diff --git a/test/secrets/aws_secrets_manager_adapter_test.rb b/test/secrets/aws_secrets_manager_adapter_test.rb index eb425554..42a0f48a 100644 --- a/test/secrets/aws_secrets_manager_adapter_test.rb +++ b/test/secrets/aws_secrets_manager_adapter_test.rb @@ -2,6 +2,7 @@ require "test_helper" class AwsSecretsManagerAdapterTest < SecretAdapterTestCase test "fetch" do + stub_ticks.with("aws --version 2> /dev/null") stub_ticks .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 secret2/KEY3 --profile default") .returns(<<~JSON) @@ -44,6 +45,7 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase end test "fetch with secret names" do + stub_ticks.with("aws --version 2> /dev/null") stub_ticks .with("aws secretsmanager batch-get-secret-value --secret-id-list secret/KEY1 secret/KEY2 --profile default") .returns(<<~JSON) @@ -74,6 +76,15 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase assert_equal expected_json, json end + test "fetch without CLI installed" do + stub_ticks_with("aws --version 2> /dev/null", succeed: false) + + error = assert_raises RuntimeError do + JSON.parse(shellunescape(run_command("fetch", "SECRET1"))) + end + assert_equal "AWS CLI is not installed", error.message + end + private def run_command(*command) stdouted do From b4d395cec9247e35c8bb67cf7d247add486c9bb5 Mon Sep 17 00:00:00 2001 From: Justin Dell Date: Mon, 4 Nov 2024 09:46:45 -0600 Subject: [PATCH 07/26] shell escape account name in cli command --- lib/kamal/secrets/adapters/aws_secrets_manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kamal/secrets/adapters/aws_secrets_manager.rb b/lib/kamal/secrets/adapters/aws_secrets_manager.rb index 1da48b94..e23ea1f1 100644 --- a/lib/kamal/secrets/adapters/aws_secrets_manager.rb +++ b/lib/kamal/secrets/adapters/aws_secrets_manager.rb @@ -18,7 +18,7 @@ class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Ba end def get_from_secrets_manager(secrets, account:) - `aws secretsmanager batch-get-secret-value --secret-id-list #{secrets.map(&:shellescape).join(" ")} --profile #{account}`.tap do + `aws secretsmanager batch-get-secret-value --secret-id-list #{secrets.map(&:shellescape).join(" ")} --profile #{account.shellescape}`.tap do raise RuntimeError, "Could not read #{secret} from AWS Secrets Manager" unless $?.success? end end From 77cd29f5ad3abddbbe46faef19b01e89892a13c9 Mon Sep 17 00:00:00 2001 From: Ralf Schmitz Bongiolo Date: Mon, 4 Nov 2024 18:58:18 -0400 Subject: [PATCH 08/26] feat(cli): update secrets --account flag as optional depending on adapter --- lib/kamal/cli/secrets.rb | 12 +++++++++--- lib/kamal/secrets/adapters/base.rb | 9 ++++++++- .../secrets/adapters/test_optional_account.rb | 18 ++++++++++++++++++ test/cli/secrets_test.rb | 12 ++++++++++++ 4 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 lib/kamal/secrets/adapters/test_optional_account.rb diff --git a/lib/kamal/cli/secrets.rb b/lib/kamal/cli/secrets.rb index b094be46..0cfcf628 100644 --- a/lib/kamal/cli/secrets.rb +++ b/lib/kamal/cli/secrets.rb @@ -1,11 +1,17 @@ class Kamal::Cli::Secrets < Kamal::Cli::Base desc "fetch [SECRETS...]", "Fetch secrets from a vault" option :adapter, type: :string, aliases: "-a", required: true, desc: "Which vault adapter to use" - option :account, type: :string, required: true, desc: "The account identifier or username" + option :account, type: :string, required: false, desc: "The account identifier or username" option :from, type: :string, required: false, desc: "A vault or folder to fetch the secrets from" option :inline, type: :boolean, required: false, hidden: true def fetch(*secrets) - results = adapter(options[:adapter]).fetch(secrets, **options.slice(:account, :from).symbolize_keys) + adapter = initialize_adapter(options[:adapter]) + + if adapter.requires_account? && options[:account].blank? + return puts "No value provided for required options '--account'" + end + + results = adapter.fetch(secrets, **options.slice(:account, :from).symbolize_keys) return_or_puts JSON.dump(results).shellescape, inline: options[:inline] end @@ -29,7 +35,7 @@ class Kamal::Cli::Secrets < Kamal::Cli::Base end private - def adapter(adapter) + def initialize_adapter(adapter) Kamal::Secrets::Adapters.lookup(adapter) end diff --git a/lib/kamal/secrets/adapters/base.rb b/lib/kamal/secrets/adapters/base.rb index 579414af..fc66bb34 100644 --- a/lib/kamal/secrets/adapters/base.rb +++ b/lib/kamal/secrets/adapters/base.rb @@ -1,13 +1,20 @@ class Kamal::Secrets::Adapters::Base delegate :optionize, to: Kamal::Utils - def fetch(secrets, account:, from: nil) + def fetch(secrets, account: nil, from: nil) + raise RuntimeError, "Missing required option '--account'" if requires_account? && account.blank? + check_dependencies! + session = login(account) full_secrets = secrets.map { |secret| [ from, secret ].compact.join("/") } fetch_secrets(full_secrets, account: account, session: session) end + def requires_account? + true + end + private def login(...) raise NotImplementedError diff --git a/lib/kamal/secrets/adapters/test_optional_account.rb b/lib/kamal/secrets/adapters/test_optional_account.rb new file mode 100644 index 00000000..1f85302f --- /dev/null +++ b/lib/kamal/secrets/adapters/test_optional_account.rb @@ -0,0 +1,18 @@ +class Kamal::Secrets::Adapters::TestOptionalAccount < Kamal::Secrets::Adapters::Base + def requires_account? + false + end + + private + def login(account) + true + end + + def fetch_secrets(secrets, account:, session:) + secrets.to_h { |secret| [ secret, secret.reverse ] } + end + + def check_dependencies! + # no op + end +end diff --git a/test/cli/secrets_test.rb b/test/cli/secrets_test.rb index 6014a7e7..bd412862 100644 --- a/test/cli/secrets_test.rb +++ b/test/cli/secrets_test.rb @@ -7,6 +7,18 @@ class CliSecretsTest < CliTestCase run_command("fetch", "foo", "bar", "baz", "--account", "myaccount", "--adapter", "test") end + test "fetch missing --acount" do + assert_equal \ + "No value provided for required options '--account'", + run_command("fetch", "foo", "bar", "baz", "--adapter", "test") + end + + test "fetch without required --account" do + assert_equal \ + "\\{\\\"foo\\\":\\\"oof\\\",\\\"bar\\\":\\\"rab\\\",\\\"baz\\\":\\\"zab\\\"\\}", + run_command("fetch", "foo", "bar", "baz", "--adapter", "test_optional_account") + end + test "extract" do assert_equal "oof", run_command("extract", "foo", "{\"foo\":\"oof\", \"bar\":\"rab\", \"baz\":\"zab\"}") end From 3069552315620af71b1540f63d093164346e2e0f Mon Sep 17 00:00:00 2001 From: Ralf Schmitz Bongiolo Date: Mon, 4 Nov 2024 19:00:38 -0400 Subject: [PATCH 09/26] feat(secrets): update doppler adapter to use --from option and DOPPLER_TOKEN env --- lib/kamal/secrets/adapters/doppler.rb | 41 ++++++++-- test/secrets/doppler_adapter_test.rb | 108 +++++++++++++++++++++++--- 2 files changed, 130 insertions(+), 19 deletions(-) diff --git a/lib/kamal/secrets/adapters/doppler.rb b/lib/kamal/secrets/adapters/doppler.rb index caa1833e..64d644f7 100644 --- a/lib/kamal/secrets/adapters/doppler.rb +++ b/lib/kamal/secrets/adapters/doppler.rb @@ -1,28 +1,53 @@ class Kamal::Secrets::Adapters::Doppler < Kamal::Secrets::Adapters::Base + def requires_account? + false + end + private - def login(account) - unless loggedin?(account) + def login(*) + unless loggedin? `doppler login -y` raise RuntimeError, "Failed to login to Doppler" unless $?.success? end end - def loggedin?(account) + def loggedin? `doppler me --json 2> /dev/null` $?.success? end - def fetch_secrets(secrets, account:, session:) - project, config = account.split("/") + def fetch_secrets(secrets, **) + project_and_config_flags = "" + unless service_token_set? + project, config, _ = secrets.first.split("/") - raise RuntimeError, "Missing project or config from --acount=project/config option" unless project && config - raise RuntimeError, "Using --from option or FOLDER/SECRET is not supported by Doppler" if secrets.any?(/\//) + unless project && config + raise RuntimeError, "Missing project or config from '--from=project/config' option" + end - items = `doppler secrets get #{secrets.map(&:shellescape).join(" ")} --json -p #{project} -c #{config}` + project_and_config_flags = "-p #{project.shellescape} -c #{config.shellescape}" + end + + secret_names = secrets.collect { |s| s.split("/").last } + + items = `doppler secrets get #{secret_names.map(&:shellescape).join(" ")} --json #{project_and_config_flags}` raise RuntimeError, "Could not read #{secrets} from Doppler" unless $?.success? items = JSON.parse(items) items.transform_values { |value| value["computed"] } end + + def service_token_set? + ENV["DOPPLER_TOKEN"] && ENV["DOPPLER_TOKEN"][0, 5] == "dp.st" + end + + def check_dependencies! + raise RuntimeError, "Doppler CLI is not installed" unless cli_installed? + end + + def cli_installed? + `doppler --version 2> /dev/null` + $?.success? + end end diff --git a/test/secrets/doppler_adapter_test.rb b/test/secrets/doppler_adapter_test.rb index c7cda494..c2b16468 100644 --- a/test/secrets/doppler_adapter_test.rb +++ b/test/secrets/doppler_adapter_test.rb @@ -6,6 +6,7 @@ class DopplerAdapterTest < SecretAdapterTestCase end test "fetch" do + stub_ticks_with("doppler --version 2> /dev/null", succeed: true) stub_ticks.with("doppler me --json 2> /dev/null") stub_ticks @@ -30,7 +31,9 @@ class DopplerAdapterTest < SecretAdapterTestCase } JSON - json = JSON.parse(shellunescape(run_command("fetch", "SECRET1", "FSECRET1", "FSECRET2"))) + json = JSON.parse( + shellunescape run_command("fetch", "--from", "my-project/prd", "SECRET1", "FSECRET1", "FSECRET2") + ) expected_json = { "SECRET1"=>"secret1", @@ -41,32 +44,106 @@ class DopplerAdapterTest < SecretAdapterTestCase assert_equal expected_json, json end - test "fetch with from" do + test "fetch having DOPPLER_TOKEN" do + ENV["DOPPLER_TOKEN"] = "dp.st.xxxxxxxxxxxxxxxxxxxxxx" + + stub_ticks_with("doppler --version 2> /dev/null", succeed: true) stub_ticks.with("doppler me --json 2> /dev/null") - error = assert_raises RuntimeError do - run_command("fetch", "--from", "FOLDER1", "FSECRET1", "FSECRET2") - end + stub_ticks + .with("doppler secrets get SECRET1 FSECRET1 FSECRET2 --json ") + .returns(<<~JSON) + { + "SECRET1": { + "computed":"secret1", + "computedVisibility":"unmasked", + "note":"" + }, + "FSECRET1": { + "computed":"fsecret1", + "computedVisibility":"unmasked", + "note":"" + }, + "FSECRET2": { + "computed":"fsecret2", + "computedVisibility":"unmasked", + "note":"" + } + } + JSON - assert_match(/Using --from option or FOLDER\/SECRET is not supported by Doppler/, error.message) + json = JSON.parse( + shellunescape run_command("fetch", "SECRET1", "FSECRET1", "FSECRET2") + ) + + expected_json = { + "SECRET1"=>"secret1", + "FSECRET1"=>"fsecret1", + "FSECRET2"=>"fsecret2" + } + + assert_equal expected_json, json + + ENV.delete("DOPPLER_TOKEN") end test "fetch with folder in secret" do + stub_ticks_with("doppler --version 2> /dev/null", succeed: true) + stub_ticks.with("doppler me --json 2> /dev/null") + + stub_ticks + .with("doppler secrets get SECRET1 FSECRET1 FSECRET2 --json -p my-project -c prd") + .returns(<<~JSON) + { + "SECRET1": { + "computed":"secret1", + "computedVisibility":"unmasked", + "note":"" + }, + "FSECRET1": { + "computed":"fsecret1", + "computedVisibility":"unmasked", + "note":"" + }, + "FSECRET2": { + "computed":"fsecret2", + "computedVisibility":"unmasked", + "note":"" + } + } + JSON + + json = JSON.parse( + shellunescape run_command("fetch", "my-project/prd/SECRET1", "my-project/prd/FSECRET1", "my-project/prd/FSECRET2") + ) + + expected_json = { + "SECRET1"=>"secret1", + "FSECRET1"=>"fsecret1", + "FSECRET2"=>"fsecret2" + } + + assert_equal expected_json, json + end + + test "fetch without --from" do + stub_ticks_with("doppler --version 2> /dev/null", succeed: true) stub_ticks.with("doppler me --json 2> /dev/null") error = assert_raises RuntimeError do - run_command("fetch", "FOLDER1/FSECRET1", "SECRET2") + run_command("fetch", "FSECRET1", "FSECRET2") end - assert_match(/Using --from option or FOLDER\/SECRET is not supported by Doppler/, error.message) + assert_equal "Missing project or config from '--from=project/config' option", error.message end test "fetch with signin" do + stub_ticks_with("doppler --version 2> /dev/null", succeed: true) stub_ticks_with("doppler me --json 2> /dev/null", succeed: false) stub_ticks_with("doppler login -y", succeed: true).returns("") stub_ticks.with("doppler secrets get SECRET1 --json -p my-project -c prd").returns(single_item_json) - json = JSON.parse(shellunescape(run_command("fetch", "SECRET1"))) + json = JSON.parse(shellunescape(run_command("fetch", "--from", "my-project/prd", "SECRET1"))) expected_json = { "SECRET1"=>"secret1" @@ -75,14 +152,23 @@ class DopplerAdapterTest < SecretAdapterTestCase assert_equal expected_json, json end + test "fetch without CLI installed" do + stub_ticks_with("doppler --version 2> /dev/null", succeed: false) + + error = assert_raises RuntimeError do + JSON.parse(shellunescape(run_command("fetch", "HOST", "PORT"))) + end + + assert_equal "Doppler CLI is not installed", error.message + end + private def run_command(*command) stdouted do Kamal::Cli::Secrets.start \ [ *command, "-c", "test/fixtures/deploy_with_accessories.yml", - "--adapter", "doppler", - "--account", "my-project/prd" ] + "--adapter", "doppler" ] end end From e4ab2a0d2433d96c450fc754b9d83de28dc16e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Wn=C4=99trzak?= Date: Tue, 5 Nov 2024 14:41:40 +0100 Subject: [PATCH 10/26] Improve error on unknown role in accessories config. Previously when unknown role (or with typo) was placed in accessories.roles, this error was thrown: `ERROR (NoMethodError): undefined method `hosts' for nil`. --- lib/kamal/configuration/accessory.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/kamal/configuration/accessory.rb b/lib/kamal/configuration/accessory.rb index 3258c9d0..aeb5f334 100644 --- a/lib/kamal/configuration/accessory.rb +++ b/lib/kamal/configuration/accessory.rb @@ -176,7 +176,9 @@ class Kamal::Configuration::Accessory def hosts_from_roles if accessory_config.key?("roles") - accessory_config["roles"].flat_map { |role| config.role(role).hosts } + accessory_config["roles"].flat_map do |role| + config.role(role)&.hosts || raise(Kamal::ConfigurationError, "Unknown role in accessories config: '#{role}'") + end end end From 8dd864af89e8e6875a783357310a4200b4a014ae Mon Sep 17 00:00:00 2001 From: Ralf Schmitz Bongiolo Date: Tue, 5 Nov 2024 14:14:18 -0400 Subject: [PATCH 11/26] refactor(secrets): adapter/test_optional_account inherit from adapter/test --- .../secrets/adapters/test_optional_account.rb | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/lib/kamal/secrets/adapters/test_optional_account.rb b/lib/kamal/secrets/adapters/test_optional_account.rb index 1f85302f..3a252e68 100644 --- a/lib/kamal/secrets/adapters/test_optional_account.rb +++ b/lib/kamal/secrets/adapters/test_optional_account.rb @@ -1,18 +1,5 @@ -class Kamal::Secrets::Adapters::TestOptionalAccount < Kamal::Secrets::Adapters::Base +class Kamal::Secrets::Adapters::TestOptionalAccount < Kamal::Secrets::Adapters::Test def requires_account? false end - - private - def login(account) - true - end - - def fetch_secrets(secrets, account:, session:) - secrets.to_h { |secret| [ secret, secret.reverse ] } - end - - def check_dependencies! - # no op - end end From c970ceebe3137932bf1198112510830d19de6cec Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Mon, 18 Nov 2024 13:01:53 -0700 Subject: [PATCH 12/26] Adds support for SBOM attestations --- lib/kamal/commands/builder/base.rb | 8 ++++++-- lib/kamal/configuration/builder.rb | 4 ++++ lib/kamal/configuration/docs/builder.yml | 6 ++++++ test/commands/builder_test.rb | 14 ++++++++++++++ test/configuration/builder_test.rb | 10 ++++++++++ 5 files changed, 40 insertions(+), 2 deletions(-) diff --git a/lib/kamal/commands/builder/base.rb b/lib/kamal/commands/builder/base.rb index d551520b..dea04a3a 100644 --- a/lib/kamal/commands/builder/base.rb +++ b/lib/kamal/commands/builder/base.rb @@ -6,7 +6,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base delegate :argumentize, to: Kamal::Utils delegate \ :args, :secrets, :dockerfile, :target, :arches, :local_arches, :remote_arches, :remote, - :cache_from, :cache_to, :ssh, :provenance, :driver, :docker_driver?, + :cache_from, :cache_to, :ssh, :provenance, :sbom, :driver, :docker_driver?, to: :builder_config def clean @@ -37,7 +37,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base end def build_options - [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_target, *build_ssh, *builder_provenance ] + [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile, *build_target, *build_ssh, *builder_provenance, *builder_sbom ] end def build_context @@ -101,6 +101,10 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base argumentize "--provenance", provenance unless provenance.nil? end + def builder_sbom + argumentize "--sbom", sbom unless sbom.nil? + end + def builder_config config.builder end diff --git a/lib/kamal/configuration/builder.rb b/lib/kamal/configuration/builder.rb index 4c0dc603..970c47d1 100644 --- a/lib/kamal/configuration/builder.rb +++ b/lib/kamal/configuration/builder.rb @@ -115,6 +115,10 @@ class Kamal::Configuration::Builder builder_config["provenance"] end + def sbom + builder_config["sbom"] + end + def git_clone? Kamal::Git.used? && builder_config["context"].nil? end diff --git a/lib/kamal/configuration/docs/builder.yml b/lib/kamal/configuration/docs/builder.yml index b6e639f3..230b39ee 100644 --- a/lib/kamal/configuration/docs/builder.yml +++ b/lib/kamal/configuration/docs/builder.yml @@ -108,3 +108,9 @@ builder: # It is used to configure provenance attestations for the build result. # The value can also be a boolean to enable or disable provenance attestations. provenance: mode=max + + # SBOM (Software Bill of Materials) + # + # It is used to configure SBOM generation for the build result. + # The value can also be a boolean to enable or disable SBOM generation. + sbom: true diff --git a/test/commands/builder_test.rb b/test/commands/builder_test.rb index 86b2f573..85703f54 100644 --- a/test/commands/builder_test.rb +++ b/test/commands/builder_test.rb @@ -158,6 +158,20 @@ class CommandsBuilderTest < ActiveSupport::TestCase builder.push.join(" ") end + test "push with sbom" do + builder = new_builder_command(builder: { "sbom" => true }) + assert_equal \ + "docker buildx build --push --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --sbom true .", + builder.push.join(" ") + end + + test "push with sbom false" do + builder = new_builder_command(builder: { "sbom" => false }) + assert_equal \ + "docker buildx build --push --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile --sbom false .", + builder.push.join(" ") + end + test "mirror count" do command = new_builder_command assert_equal "docker info --format '{{index .RegistryConfig.Mirrors 0}}'", command.first_mirror.join(" ") diff --git a/test/configuration/builder_test.rb b/test/configuration/builder_test.rb index 5fef465a..12387800 100644 --- a/test/configuration/builder_test.rb +++ b/test/configuration/builder_test.rb @@ -144,6 +144,16 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase assert_equal "mode=max", config.builder.provenance end + test "sbom" do + assert_nil config.builder.sbom + end + + test "setting sbom" do + @deploy[:builder]["sbom"] = true + + assert_equal true, config.builder.sbom + end + test "local disabled but no remote set" do @deploy[:builder]["local"] = false From 72f30774ba341e6a4b75b0ced4c9cc4925427c3a Mon Sep 17 00:00:00 2001 From: Lewis Buckley Date: Wed, 20 Nov 2024 11:56:58 +0000 Subject: [PATCH 13/26] Support line filtering when running tests --- test/test_helper.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_helper.rb b/test/test_helper.rb index 1749ee32..f5811872 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -2,6 +2,7 @@ require "bundler/setup" require "active_support/test_case" require "active_support/testing/autorun" require "active_support/testing/stream" +require "rails/test_unit/line_filtering" require "debug" require "mocha/minitest" # using #stubs that can alter returns require "minitest/autorun" # using #stub that take args @@ -32,6 +33,7 @@ end class ActiveSupport::TestCase include ActiveSupport::Testing::Stream + extend Rails::LineFiltering private def stdouted From 83fd2a051d495716db30c2515f16fb14b5a45976 Mon Sep 17 00:00:00 2001 From: Matteo Giaccone Date: Thu, 21 Nov 2024 11:06:36 +0100 Subject: [PATCH 14/26] Add support for exec output in accessories When running accessory exec now you get the output from the hosts. Also you can pass commands with arguments and it will work e.g.: cat yourfilename --- lib/kamal/cli/accessory.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/kamal/cli/accessory.rb b/lib/kamal/cli/accessory.rb index 6c51c774..d707820a 100644 --- a/lib/kamal/cli/accessory.rb +++ b/lib/kamal/cli/accessory.rb @@ -112,14 +112,15 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base end end - desc "exec [NAME] [CMD]", "Execute a custom command on servers (use --help to show options)" + desc "exec [NAME] [CMD...]", "Execute a custom command on servers within the accessory container (use --help to show options)" option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)" option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one" - def exec(name, cmd) + def exec(name, *cmd) + cmd = Kamal::Utils.join_commands(cmd) with_accessory(name) do |accessory, hosts| case when options[:interactive] && options[:reuse] - say "Launching interactive command with via SSH from existing container...", :magenta + say "Launching interactive command via SSH from existing container...", :magenta run_locally { exec accessory.execute_in_existing_container_over_ssh(cmd) } when options[:interactive] @@ -128,16 +129,16 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base when options[:reuse] say "Launching command from existing container...", :magenta - on(hosts) do + on(hosts) do |host| execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug - capture_with_info(*accessory.execute_in_existing_container(cmd)) + puts_by_host host, capture_with_info(*accessory.execute_in_existing_container(cmd)) end else say "Launching command from new container...", :magenta - on(hosts) do + on(hosts) do |host| execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on #{name} accessory"), verbosity: :debug - capture_with_info(*accessory.execute_in_new_container(cmd)) + puts_by_host host, capture_with_info(*accessory.execute_in_new_container(cmd)) end end end From 70d2c71734c6778fa22d91e48a91a641b9129c48 Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Thu, 31 Oct 2024 10:50:48 +0400 Subject: [PATCH 15/26] Added commands to deploy accessory to kamal-proxy --- lib/kamal/cli/accessory.rb | 3 +++ lib/kamal/commands/accessory.rb | 18 +++++++++++++++++- lib/kamal/configuration/accessory.rb | 15 ++++++++++++++- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/lib/kamal/cli/accessory.rb b/lib/kamal/cli/accessory.rb index d707820a..88589049 100644 --- a/lib/kamal/cli/accessory.rb +++ b/lib/kamal/cli/accessory.rb @@ -18,6 +18,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base execute *accessory.ensure_env_directory upload! accessory.secrets_io, accessory.secrets_path, mode: "0600" execute *accessory.run + execute *accessory.deploy if accessory.running_proxy? end end end @@ -75,6 +76,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base on(hosts) do execute *KAMAL.auditor.record("Started #{name} accessory"), verbosity: :debug execute *accessory.start + execute *accessory.deploy if accessory.running_proxy? end end end @@ -87,6 +89,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base on(hosts) do execute *KAMAL.auditor.record("Stopped #{name} accessory"), verbosity: :debug execute *accessory.stop, raise_on_non_zero_exit: false + # execute *accessory.remove if accessory.running_proxy? end end end diff --git a/lib/kamal/commands/accessory.rb b/lib/kamal/commands/accessory.rb index 9abb6dfc..9068ba9b 100644 --- a/lib/kamal/commands/accessory.rb +++ b/lib/kamal/commands/accessory.rb @@ -2,8 +2,10 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base attr_reader :accessory_config delegate :service_name, :image, :hosts, :port, :files, :directories, :cmd, :network_args, :publish_args, :env_args, :volume_args, :label_args, :option_args, - :secrets_io, :secrets_path, :env_directory, + :secrets_io, :secrets_path, :env_directory, :running_proxy?, to: :accessory_config + delegate :proxy_container_name, to: :config + def initialize(config, name:) super(config) @@ -38,6 +40,16 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base docker :ps, *service_filter end + def deploy + target = container_id_for(container_name: service_name, only_running: true) + proxy_exec :deploy, service_name, *accessory_config.proxy.deploy_command_args(target: target) if target + end + + def remove + target = container_id_for(container_name: service_name, only_running: true) + proxy_exec :remove, service_name, *accessory_config.proxy.remove_command_args(target: target) if target + end + def logs(timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil) pipe \ @@ -110,4 +122,8 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base def service_filter [ "--filter", "label=service=#{service_name}" ] end + + def proxy_exec(*command) + docker :exec, proxy_container_name, "kamal-proxy", *command + end end diff --git a/lib/kamal/configuration/accessory.rb b/lib/kamal/configuration/accessory.rb index aeb5f334..2203e64c 100644 --- a/lib/kamal/configuration/accessory.rb +++ b/lib/kamal/configuration/accessory.rb @@ -5,7 +5,7 @@ class Kamal::Configuration::Accessory delegate :argumentize, :optionize, to: Kamal::Utils - attr_reader :name, :accessory_config, :env + attr_reader :name, :accessory_config, :env, :proxy def initialize(name, config:) @name, @config, @accessory_config = name.inquiry, config, config.raw_config["accessories"][name] @@ -20,6 +20,8 @@ class Kamal::Configuration::Accessory config: accessory_config.fetch("env", {}), secrets: config.secrets, context: "accessories/#{name}/env" + + # initialize_proxy if running_proxy? end def service_name @@ -106,6 +108,17 @@ class Kamal::Configuration::Accessory accessory_config["cmd"] end + def running_proxy? + @accessory_config["proxy"].present? + end + + def initialize_proxy + @proxy = Kamal::Configuration::Proxy.new \ + config: config, + proxy_config: accessory_config["proxy"], + context: "accessories/#{name}/proxy" + end + private attr_accessor :config From 4c778de2d93ef1b5e8e7e71174003b7467d43d80 Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Thu, 31 Oct 2024 10:51:51 +0400 Subject: [PATCH 16/26] Added tests for accessory configuration with proxy --- lib/kamal/cli/accessory.rb | 2 +- lib/kamal/configuration/accessory.rb | 2 +- lib/kamal/configuration/docs/accessory.yml | 84 ++++++++++++++++++++++ test/configuration/accessory_test.rb | 8 +++ 4 files changed, 94 insertions(+), 2 deletions(-) diff --git a/lib/kamal/cli/accessory.rb b/lib/kamal/cli/accessory.rb index 88589049..e37e4116 100644 --- a/lib/kamal/cli/accessory.rb +++ b/lib/kamal/cli/accessory.rb @@ -89,7 +89,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base on(hosts) do execute *KAMAL.auditor.record("Stopped #{name} accessory"), verbosity: :debug execute *accessory.stop, raise_on_non_zero_exit: false - # execute *accessory.remove if accessory.running_proxy? + execute *accessory.remove if accessory.running_proxy? end end end diff --git a/lib/kamal/configuration/accessory.rb b/lib/kamal/configuration/accessory.rb index 2203e64c..2728607d 100644 --- a/lib/kamal/configuration/accessory.rb +++ b/lib/kamal/configuration/accessory.rb @@ -21,7 +21,7 @@ class Kamal::Configuration::Accessory secrets: config.secrets, context: "accessories/#{name}/env" - # initialize_proxy if running_proxy? + initialize_proxy if running_proxy? end def service_name diff --git a/lib/kamal/configuration/docs/accessory.yml b/lib/kamal/configuration/docs/accessory.yml index e77bf754..f7e78c06 100644 --- a/lib/kamal/configuration/docs/accessory.yml +++ b/lib/kamal/configuration/docs/accessory.yml @@ -98,3 +98,87 @@ accessories: # Defaults to kamal: network: custom + # Proxy + # + proxy: + # Host + # + # The hosts that will be used to serve the app. The proxy will only route requests + # to this host to your app. + # + # If no hosts are set, then all requests will be forwarded, except for matching + # requests for other apps deployed on that server that do have a host set. + host: foo.example.com + + # App port + # + # The port the application container is exposed on + # + # Defaults to 80 + app_port: 3000 + + # SSL + # + # kamal-proxy can provide automatic HTTPS for your application via Let's Encrypt. + # + # This requires that we are deploying to a one server and the host option is set. + # The host value must point to the server we are deploying to and port 443 must be + # open for the Let's Encrypt challenge to succeed. + # + # Defaults to false + ssl: true + + # Response timeout + # + # How long to wait for requests to complete before timing out, defaults to 30 seconds + response_timeout: 10 + + # Healthcheck + # + # When deploying, the proxy will by default hit /up once every second until we hit + # the deploy timeout, with a 5 second timeout for each request. + # + # Once the app is up, the proxy will stop hitting the healthcheck endpoint. + healthcheck: + interval: 3 + path: /health + timeout: 3 + + # Buffering + # + # Whether to buffer request and response bodies in the proxy + # + # By default buffering is enabled with a max request body size of 1GB and no limit + # for response size. + # + # You can also set the memory limit for buffering, which defaults to 1MB, anything + # larger than that is written to disk. + buffering: + requests: true + responses: true + max_request_body: 40_000_000 + max_response_body: 0 + memory: 2_000_000 + + # Logging + # + # Configure request logging for the proxy + # You can specify request and response headers to log. + # By default, Cache-Control, Last-Modified and User-Agent request headers are logged + logging: + request_headers: + - Cache-Control + - X-Forwarded-Proto + response_headers: + - X-Request-ID + - X-Request-Start + + # Forward headers + # + # Whether to forward the X-Forwarded-For and X-Forwarded-Proto headers. + # + # If you are behind a trusted proxy, you can set this to true to forward the headers. + # + # By default kamal-proxy will not forward the headers the ssl option is set to true, and + # will forward them if it is set to false. + forward_headers: true diff --git a/test/configuration/accessory_test.rb b/test/configuration/accessory_test.rb index f5220902..9440eca6 100644 --- a/test/configuration/accessory_test.rb +++ b/test/configuration/accessory_test.rb @@ -63,6 +63,9 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase "options" => { "cpus" => "4", "memory" => "2GB" + }, + "proxy" => { + "host" => "monitoring.example.com" } } } @@ -161,4 +164,9 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase @deploy[:accessories]["mysql"]["network"] = "database" assert_equal [ "--network", "database" ], @config.accessory(:mysql).network_args end + + test "proxy" do + assert @config.accessory(:monitoring).running_proxy? + assert_equal "monitoring.example.com", @config.accessory(:monitoring).proxy.host + end end From f4b7c886fbf1fa4c662cc2b3dd22a66c8664d36d Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Fri, 27 Sep 2024 12:24:54 +0400 Subject: [PATCH 17/26] Added tests for accessory deploy and remove commands --- lib/kamal/cli/accessory.rb | 17 ++++++++++++++--- lib/kamal/commands/accessory.rb | 10 ++++------ test/commands/accessory_test.rb | 17 ++++++++++++++++- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/lib/kamal/cli/accessory.rb b/lib/kamal/cli/accessory.rb index e37e4116..70aa208c 100644 --- a/lib/kamal/cli/accessory.rb +++ b/lib/kamal/cli/accessory.rb @@ -18,7 +18,11 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base execute *accessory.ensure_env_directory upload! accessory.secrets_io, accessory.secrets_path, mode: "0600" execute *accessory.run - execute *accessory.deploy if accessory.running_proxy? + + if accessory.running_proxy? + target = accessory.container_id_for(container_name: accessory.service_name, only_running: true) + execute *accessory.deploy(target: target) + end end end end @@ -76,7 +80,10 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base on(hosts) do execute *KAMAL.auditor.record("Started #{name} accessory"), verbosity: :debug execute *accessory.start - execute *accessory.deploy if accessory.running_proxy? + if accessory.running_proxy? + target = container_id_for(container_name: service_name, only_running: true) + execute *accessory.deploy(target: target) + end end end end @@ -89,7 +96,11 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base on(hosts) do execute *KAMAL.auditor.record("Stopped #{name} accessory"), verbosity: :debug execute *accessory.stop, raise_on_non_zero_exit: false - execute *accessory.remove if accessory.running_proxy? + + if accessory.running_proxy? + target = accessory.container_id_for(container_name: accessory.service_name, only_running: true) + execute *accessory.remove(target: target) + end end end end diff --git a/lib/kamal/commands/accessory.rb b/lib/kamal/commands/accessory.rb index 9068ba9b..e002b28d 100644 --- a/lib/kamal/commands/accessory.rb +++ b/lib/kamal/commands/accessory.rb @@ -40,14 +40,12 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base docker :ps, *service_filter end - def deploy - target = container_id_for(container_name: service_name, only_running: true) - proxy_exec :deploy, service_name, *accessory_config.proxy.deploy_command_args(target: target) if target + def deploy(target:) + proxy_exec :deploy, service_name, *accessory_config.proxy.deploy_command_args(target: target) end - def remove - target = container_id_for(container_name: service_name, only_running: true) - proxy_exec :remove, service_name, *accessory_config.proxy.remove_command_args(target: target) if target + def remove(target:) + proxy_exec :remove, service_name, *accessory_config.proxy.remove_command_args(target: target) end diff --git a/test/commands/accessory_test.rb b/test/commands/accessory_test.rb index 1befd9e6..9909de3a 100644 --- a/test/commands/accessory_test.rb +++ b/test/commands/accessory_test.rb @@ -39,7 +39,10 @@ class CommandsAccessoryTest < ActiveSupport::TestCase "busybox" => { "service" => "custom-busybox", "image" => "busybox:latest", - "host" => "1.1.1.7" + "host" => "1.1.1.7", + "proxy" => { + "host" => "busybox.example.com" + } } } } @@ -166,6 +169,18 @@ class CommandsAccessoryTest < ActiveSupport::TestCase new_command(:mysql).remove_image.join(" ") end + test "deploy" do + assert_equal \ + "docker exec kamal-proxy kamal-proxy deploy custom-busybox --target \"172.1.0.2:80\" --deploy-timeout \"30s\" --drain-timeout \"30s\" --buffer-requests --buffer-responses --log-request-header \"Cache-Control\" --log-request-header \"Last-Modified\" --log-request-header \"User-Agent\"", + new_command(:busybox).deploy(target: "172.1.0.2").join(" ") + end + + test "remove" do + assert_equal \ + "docker exec kamal-proxy kamal-proxy remove custom-busybox --target \"172.1.0.2:80\"", + new_command(:busybox).remove(target: "172.1.0.2").join(" ") + end + private def new_command(accessory) Kamal::Commands::Accessory.new(Kamal::Configuration.new(@config), name: accessory) From aa2906086a458e607f669b04c6e20ae9180a0992 Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Fri, 27 Sep 2024 12:38:12 +0400 Subject: [PATCH 18/26] Added host to the expected accessory deploy command result --- test/commands/accessory_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/commands/accessory_test.rb b/test/commands/accessory_test.rb index 9909de3a..4da4669a 100644 --- a/test/commands/accessory_test.rb +++ b/test/commands/accessory_test.rb @@ -171,7 +171,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase test "deploy" do assert_equal \ - "docker exec kamal-proxy kamal-proxy deploy custom-busybox --target \"172.1.0.2:80\" --deploy-timeout \"30s\" --drain-timeout \"30s\" --buffer-requests --buffer-responses --log-request-header \"Cache-Control\" --log-request-header \"Last-Modified\" --log-request-header \"User-Agent\"", + "docker exec kamal-proxy kamal-proxy deploy custom-busybox --target \"172.1.0.2:80\" --host \"busybox.example.com\" --deploy-timeout \"30s\" --drain-timeout \"30s\" --buffer-requests --buffer-responses --log-request-header \"Cache-Control\" --log-request-header \"Last-Modified\" --log-request-header \"User-Agent\"", new_command(:busybox).deploy(target: "172.1.0.2").join(" ") end From 86657b0172e68460aeb70ed3e4856cdbe2cc594a Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Thu, 31 Oct 2024 10:56:43 +0400 Subject: [PATCH 19/26] Fixed kamal-proxy remove command --- lib/kamal/cli/accessory.rb | 4 ++-- lib/kamal/commands/accessory.rb | 8 ++++---- test/commands/accessory_test.rb | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/kamal/cli/accessory.rb b/lib/kamal/cli/accessory.rb index 70aa208c..4cebed91 100644 --- a/lib/kamal/cli/accessory.rb +++ b/lib/kamal/cli/accessory.rb @@ -98,8 +98,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base execute *accessory.stop, raise_on_non_zero_exit: false if accessory.running_proxy? - target = accessory.container_id_for(container_name: accessory.service_name, only_running: true) - execute *accessory.remove(target: target) + target = capture_with_info(*accessory.container_id_for(container_name: accessory.service_name, only_running: true)).strip + execute *accessory.remove if target end end end diff --git a/lib/kamal/commands/accessory.rb b/lib/kamal/commands/accessory.rb index e002b28d..4db10751 100644 --- a/lib/kamal/commands/accessory.rb +++ b/lib/kamal/commands/accessory.rb @@ -2,7 +2,7 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base attr_reader :accessory_config delegate :service_name, :image, :hosts, :port, :files, :directories, :cmd, :network_args, :publish_args, :env_args, :volume_args, :label_args, :option_args, - :secrets_io, :secrets_path, :env_directory, :running_proxy?, + :secrets_io, :secrets_path, :env_directory, :proxy, :running_proxy?, to: :accessory_config delegate :proxy_container_name, to: :config @@ -41,11 +41,11 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base end def deploy(target:) - proxy_exec :deploy, service_name, *accessory_config.proxy.deploy_command_args(target: target) + proxy_exec :deploy, service_name, *proxy.deploy_command_args(target: target) end - def remove(target:) - proxy_exec :remove, service_name, *accessory_config.proxy.remove_command_args(target: target) + def remove + proxy_exec :remove, service_name end diff --git a/test/commands/accessory_test.rb b/test/commands/accessory_test.rb index 4da4669a..8a8f929a 100644 --- a/test/commands/accessory_test.rb +++ b/test/commands/accessory_test.rb @@ -177,7 +177,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase test "remove" do assert_equal \ - "docker exec kamal-proxy kamal-proxy remove custom-busybox --target \"172.1.0.2:80\"", + "docker exec kamal-proxy kamal-proxy remove custom-busybox", new_command(:busybox).remove(target: "172.1.0.2").join(" ") end From 4d8241ebab9036aa0d6839a5372b506134d17e20 Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Wed, 9 Oct 2024 12:33:22 +0400 Subject: [PATCH 20/26] Fixed kamal-proxy remove command --- test/commands/accessory_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/commands/accessory_test.rb b/test/commands/accessory_test.rb index 8a8f929a..633cfc0a 100644 --- a/test/commands/accessory_test.rb +++ b/test/commands/accessory_test.rb @@ -178,7 +178,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase test "remove" do assert_equal \ "docker exec kamal-proxy kamal-proxy remove custom-busybox", - new_command(:busybox).remove(target: "172.1.0.2").join(" ") + new_command(:busybox).remove.join(" ") end private From 006fa0de17817d74b1471517db6fbf624340c327 Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Sat, 28 Sep 2024 20:56:13 +0400 Subject: [PATCH 21/26] Extracted proxy commands to a module --- lib/kamal/commands/accessory.rb | 14 ++++++-------- lib/kamal/commands/app.rb | 10 +++++++++- lib/kamal/commands/{app/proxy.rb => proxy/exec.rb} | 6 +++--- 3 files changed, 18 insertions(+), 12 deletions(-) rename lib/kamal/commands/{app/proxy.rb => proxy/exec.rb} (54%) diff --git a/lib/kamal/commands/accessory.rb b/lib/kamal/commands/accessory.rb index 4db10751..cc72da8e 100644 --- a/lib/kamal/commands/accessory.rb +++ b/lib/kamal/commands/accessory.rb @@ -1,4 +1,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base + include Kamal::Commands::Proxy::Exec + attr_reader :accessory_config delegate :service_name, :image, :hosts, :port, :files, :directories, :cmd, :network_args, :publish_args, :env_args, :volume_args, :label_args, :option_args, @@ -40,14 +42,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base docker :ps, *service_filter end - def deploy(target:) - proxy_exec :deploy, service_name, *proxy.deploy_command_args(target: target) - end - - def remove - proxy_exec :remove, service_name - end - def logs(timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil) pipe \ @@ -117,6 +111,10 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base end private + def proxy_deploy_command_args(target:) + proxy.deploy_command_args(target: target) + end + def service_filter [ "--filter", "label=service=#{service_name}" ] end diff --git a/lib/kamal/commands/app.rb b/lib/kamal/commands/app.rb index 6c4df0e4..5bbbdb51 100644 --- a/lib/kamal/commands/app.rb +++ b/lib/kamal/commands/app.rb @@ -1,5 +1,5 @@ class Kamal::Commands::App < Kamal::Commands::Base - include Assets, Containers, Execution, Images, Logging, Proxy + include Assets, Containers, Execution, Images, Logging, Kamal::Commands::Proxy::Exec ACTIVE_DOCKER_STATUSES = [ :running, :restarting ] @@ -76,6 +76,14 @@ class Kamal::Commands::App < Kamal::Commands::Base end private + def service_name + role.container_prefix + end + + def proxy_deploy_command_args(target:) + role.proxy.deploy_command_args(target: target) + end + def latest_image_id docker :image, :ls, *argumentize("--filter", "reference=#{config.latest_image}"), "--format", "'{{.ID}}'" end diff --git a/lib/kamal/commands/app/proxy.rb b/lib/kamal/commands/proxy/exec.rb similarity index 54% rename from lib/kamal/commands/app/proxy.rb rename to lib/kamal/commands/proxy/exec.rb index 777a4aaf..ac6f30a9 100644 --- a/lib/kamal/commands/app/proxy.rb +++ b/lib/kamal/commands/proxy/exec.rb @@ -1,12 +1,12 @@ -module Kamal::Commands::App::Proxy +module Kamal::Commands::Proxy::Exec delegate :proxy_container_name, to: :config def deploy(target:) - proxy_exec :deploy, role.container_prefix, *role.proxy.deploy_command_args(target: target) + proxy_exec :deploy, service_name, *proxy_deploy_command_args(target: target) end def remove - proxy_exec :remove, role.container_prefix + proxy_exec :remove, service_name end private From 92046247526f2cd768e3577851db27e7b73237a5 Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Mon, 30 Sep 2024 13:07:05 +0400 Subject: [PATCH 22/26] Removed duplicated method --- lib/kamal/commands/accessory.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/kamal/commands/accessory.rb b/lib/kamal/commands/accessory.rb index cc72da8e..552a5c3b 100644 --- a/lib/kamal/commands/accessory.rb +++ b/lib/kamal/commands/accessory.rb @@ -118,8 +118,4 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base def service_filter [ "--filter", "label=service=#{service_name}" ] end - - def proxy_exec(*command) - docker :exec, proxy_container_name, "kamal-proxy", *command - end end From f52826b2d6e39d6fe50936ec1fbd9df0100d15d9 Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Wed, 9 Oct 2024 14:06:38 +0400 Subject: [PATCH 23/26] Updated accessory proxy to support hosts option --- lib/kamal/configuration/docs/accessory.yml | 7 ++++++- test/commands/accessory_test.rb | 2 +- test/configuration/accessory_test.rb | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/kamal/configuration/docs/accessory.yml b/lib/kamal/configuration/docs/accessory.yml index f7e78c06..a22cdeb0 100644 --- a/lib/kamal/configuration/docs/accessory.yml +++ b/lib/kamal/configuration/docs/accessory.yml @@ -101,14 +101,19 @@ accessories: # Proxy # proxy: - # Host + # Hosts # # The hosts that will be used to serve the app. The proxy will only route requests # to this host to your app. # # If no hosts are set, then all requests will be forwarded, except for matching # requests for other apps deployed on that server that do have a host set. + # + # Specify one of `host` or `hosts`. host: foo.example.com + hosts: + - foo.example.com + - bar.example.com # App port # diff --git a/test/commands/accessory_test.rb b/test/commands/accessory_test.rb index 633cfc0a..b9bcca7e 100644 --- a/test/commands/accessory_test.rb +++ b/test/commands/accessory_test.rb @@ -171,7 +171,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase test "deploy" do assert_equal \ - "docker exec kamal-proxy kamal-proxy deploy custom-busybox --target \"172.1.0.2:80\" --host \"busybox.example.com\" --deploy-timeout \"30s\" --drain-timeout \"30s\" --buffer-requests --buffer-responses --log-request-header \"Cache-Control\" --log-request-header \"Last-Modified\" --log-request-header \"User-Agent\"", + "docker exec kamal-proxy kamal-proxy deploy custom-busybox --target=\"172.1.0.2:80\" --host=\"busybox.example.com\" --deploy-timeout=\"30s\" --drain-timeout=\"30s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\"", new_command(:busybox).deploy(target: "172.1.0.2").join(" ") end diff --git a/test/configuration/accessory_test.rb b/test/configuration/accessory_test.rb index 9440eca6..d15a48ad 100644 --- a/test/configuration/accessory_test.rb +++ b/test/configuration/accessory_test.rb @@ -167,6 +167,6 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase test "proxy" do assert @config.accessory(:monitoring).running_proxy? - assert_equal "monitoring.example.com", @config.accessory(:monitoring).proxy.host + assert_equal [ "monitoring.example.com" ], @config.accessory(:monitoring).proxy.hosts end end From 14068b32b1a4a6114a94f4b64186d04401f46e59 Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Thu, 21 Nov 2024 22:38:06 +0400 Subject: [PATCH 24/26] Added alias to accessories proxy configuration example --- lib/kamal/configuration/docs/accessory.yml | 87 +--------------------- 1 file changed, 1 insertion(+), 86 deletions(-) diff --git a/lib/kamal/configuration/docs/accessory.yml b/lib/kamal/configuration/docs/accessory.yml index a22cdeb0..fab2989f 100644 --- a/lib/kamal/configuration/docs/accessory.yml +++ b/lib/kamal/configuration/docs/accessory.yml @@ -101,89 +101,4 @@ accessories: # Proxy # proxy: - # Hosts - # - # The hosts that will be used to serve the app. The proxy will only route requests - # to this host to your app. - # - # If no hosts are set, then all requests will be forwarded, except for matching - # requests for other apps deployed on that server that do have a host set. - # - # Specify one of `host` or `hosts`. - host: foo.example.com - hosts: - - foo.example.com - - bar.example.com - - # App port - # - # The port the application container is exposed on - # - # Defaults to 80 - app_port: 3000 - - # SSL - # - # kamal-proxy can provide automatic HTTPS for your application via Let's Encrypt. - # - # This requires that we are deploying to a one server and the host option is set. - # The host value must point to the server we are deploying to and port 443 must be - # open for the Let's Encrypt challenge to succeed. - # - # Defaults to false - ssl: true - - # Response timeout - # - # How long to wait for requests to complete before timing out, defaults to 30 seconds - response_timeout: 10 - - # Healthcheck - # - # When deploying, the proxy will by default hit /up once every second until we hit - # the deploy timeout, with a 5 second timeout for each request. - # - # Once the app is up, the proxy will stop hitting the healthcheck endpoint. - healthcheck: - interval: 3 - path: /health - timeout: 3 - - # Buffering - # - # Whether to buffer request and response bodies in the proxy - # - # By default buffering is enabled with a max request body size of 1GB and no limit - # for response size. - # - # You can also set the memory limit for buffering, which defaults to 1MB, anything - # larger than that is written to disk. - buffering: - requests: true - responses: true - max_request_body: 40_000_000 - max_response_body: 0 - memory: 2_000_000 - - # Logging - # - # Configure request logging for the proxy - # You can specify request and response headers to log. - # By default, Cache-Control, Last-Modified and User-Agent request headers are logged - logging: - request_headers: - - Cache-Control - - X-Forwarded-Proto - response_headers: - - X-Request-ID - - X-Request-Start - - # Forward headers - # - # Whether to forward the X-Forwarded-For and X-Forwarded-Proto headers. - # - # If you are behind a trusted proxy, you can set this to true to forward the headers. - # - # By default kamal-proxy will not forward the headers the ssl option is set to true, and - # will forward them if it is set to false. - forward_headers: true + ... \ No newline at end of file From f367ca8ea56dc2622cde668822a64df1673c849f Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Thu, 21 Nov 2024 23:07:55 +0400 Subject: [PATCH 25/26] Replaced Kamal::Commands::Proxy::Exec with Kamal::Commands::App::Proxy and Kamal::Commands::Accessory::Proxy --- lib/kamal/commands/accessory.rb | 6 +----- .../{proxy/exec.rb => accessory/proxy.rb} | 4 ++-- lib/kamal/commands/app.rb | 10 +--------- lib/kamal/commands/app/proxy.rb | 16 ++++++++++++++++ 4 files changed, 20 insertions(+), 16 deletions(-) rename lib/kamal/commands/{proxy/exec.rb => accessory/proxy.rb} (71%) create mode 100644 lib/kamal/commands/app/proxy.rb diff --git a/lib/kamal/commands/accessory.rb b/lib/kamal/commands/accessory.rb index 552a5c3b..281b8713 100644 --- a/lib/kamal/commands/accessory.rb +++ b/lib/kamal/commands/accessory.rb @@ -1,5 +1,5 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base - include Kamal::Commands::Proxy::Exec + include Proxy attr_reader :accessory_config delegate :service_name, :image, :hosts, :port, :files, :directories, :cmd, @@ -111,10 +111,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base end private - def proxy_deploy_command_args(target:) - proxy.deploy_command_args(target: target) - end - def service_filter [ "--filter", "label=service=#{service_name}" ] end diff --git a/lib/kamal/commands/proxy/exec.rb b/lib/kamal/commands/accessory/proxy.rb similarity index 71% rename from lib/kamal/commands/proxy/exec.rb rename to lib/kamal/commands/accessory/proxy.rb index ac6f30a9..195a321b 100644 --- a/lib/kamal/commands/proxy/exec.rb +++ b/lib/kamal/commands/accessory/proxy.rb @@ -1,8 +1,8 @@ -module Kamal::Commands::Proxy::Exec +module Kamal::Commands::Accessory::Proxy delegate :proxy_container_name, to: :config def deploy(target:) - proxy_exec :deploy, service_name, *proxy_deploy_command_args(target: target) + proxy_exec :deploy, service_name, *proxy.deploy_command_args(target: target) end def remove diff --git a/lib/kamal/commands/app.rb b/lib/kamal/commands/app.rb index 5bbbdb51..6c4df0e4 100644 --- a/lib/kamal/commands/app.rb +++ b/lib/kamal/commands/app.rb @@ -1,5 +1,5 @@ class Kamal::Commands::App < Kamal::Commands::Base - include Assets, Containers, Execution, Images, Logging, Kamal::Commands::Proxy::Exec + include Assets, Containers, Execution, Images, Logging, Proxy ACTIVE_DOCKER_STATUSES = [ :running, :restarting ] @@ -76,14 +76,6 @@ class Kamal::Commands::App < Kamal::Commands::Base end private - def service_name - role.container_prefix - end - - def proxy_deploy_command_args(target:) - role.proxy.deploy_command_args(target: target) - end - def latest_image_id docker :image, :ls, *argumentize("--filter", "reference=#{config.latest_image}"), "--format", "'{{.ID}}'" end diff --git a/lib/kamal/commands/app/proxy.rb b/lib/kamal/commands/app/proxy.rb new file mode 100644 index 00000000..777a4aaf --- /dev/null +++ b/lib/kamal/commands/app/proxy.rb @@ -0,0 +1,16 @@ +module Kamal::Commands::App::Proxy + delegate :proxy_container_name, to: :config + + def deploy(target:) + proxy_exec :deploy, role.container_prefix, *role.proxy.deploy_command_args(target: target) + end + + def remove + proxy_exec :remove, role.container_prefix + end + + private + def proxy_exec(*command) + docker :exec, proxy_container_name, "kamal-proxy", *command + end +end From eee47d10eee7738a18f6a9acb846949d373c8a53 Mon Sep 17 00:00:00 2001 From: Igor Alexandrov Date: Tue, 26 Nov 2024 13:34:51 +0400 Subject: [PATCH 26/26] Added an integration test for proxied accessory using Busybox and netcat --- lib/kamal/cli/accessory.rb | 4 +- test/integration/docker/deployer/Dockerfile | 2 + .../app_with_proxied_accessory/Dockerfile | 9 +++ .../config/deploy.yml | 44 +++++++++++++ .../app_with_proxied_accessory/default.conf | 17 +++++ test/integration/proxied_accessory_test.rb | 63 +++++++++++++++++++ 6 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 test/integration/docker/deployer/app_with_proxied_accessory/Dockerfile create mode 100644 test/integration/docker/deployer/app_with_proxied_accessory/config/deploy.yml create mode 100644 test/integration/docker/deployer/app_with_proxied_accessory/default.conf create mode 100644 test/integration/proxied_accessory_test.rb diff --git a/lib/kamal/cli/accessory.rb b/lib/kamal/cli/accessory.rb index 4cebed91..4de8832b 100644 --- a/lib/kamal/cli/accessory.rb +++ b/lib/kamal/cli/accessory.rb @@ -20,7 +20,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base execute *accessory.run if accessory.running_proxy? - target = accessory.container_id_for(container_name: accessory.service_name, only_running: true) + target = capture_with_info(*accessory.container_id_for(container_name: accessory.service_name, only_running: true)).strip execute *accessory.deploy(target: target) end end @@ -81,7 +81,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base execute *KAMAL.auditor.record("Started #{name} accessory"), verbosity: :debug execute *accessory.start if accessory.running_proxy? - target = container_id_for(container_name: service_name, only_running: true) + target = capture_with_info(*accessory.container_id_for(container_name: accessory.service_name, only_running: true)).strip execute *accessory.deploy(target: target) end end diff --git a/test/integration/docker/deployer/Dockerfile b/test/integration/docker/deployer/Dockerfile index c7132861..c25747a2 100644 --- a/test/integration/docker/deployer/Dockerfile +++ b/test/integration/docker/deployer/Dockerfile @@ -20,6 +20,7 @@ COPY *.sh . COPY app/ app/ COPY app_with_roles/ app_with_roles/ COPY app_with_traefik/ app_with_traefik/ +COPY app_with_proxied_accessory/ app_with_proxied_accessory/ RUN rm -rf /root/.ssh RUN ln -s /shared/ssh /root/.ssh @@ -30,6 +31,7 @@ RUN git config --global user.name "Deployer" RUN cd app && git init && git add . && git commit -am "Initial version" RUN cd app_with_roles && git init && git add . && git commit -am "Initial version" RUN cd app_with_traefik && git init && git add . && git commit -am "Initial version" +RUN cd app_with_proxied_accessory && git init && git add . && git commit -am "Initial version" HEALTHCHECK --interval=1s CMD pgrep sleep diff --git a/test/integration/docker/deployer/app_with_proxied_accessory/Dockerfile b/test/integration/docker/deployer/app_with_proxied_accessory/Dockerfile new file mode 100644 index 00000000..0e6237df --- /dev/null +++ b/test/integration/docker/deployer/app_with_proxied_accessory/Dockerfile @@ -0,0 +1,9 @@ +FROM registry:4443/nginx:1-alpine-slim + +COPY default.conf /etc/nginx/conf.d/default.conf + +ARG COMMIT_SHA +RUN echo $COMMIT_SHA > /usr/share/nginx/html/version +RUN mkdir -p /usr/share/nginx/html/versions && echo "version" > /usr/share/nginx/html/versions/$COMMIT_SHA +RUN mkdir -p /usr/share/nginx/html/versions && echo "hidden" > /usr/share/nginx/html/versions/.hidden +RUN echo "Up!" > /usr/share/nginx/html/up diff --git a/test/integration/docker/deployer/app_with_proxied_accessory/config/deploy.yml b/test/integration/docker/deployer/app_with_proxied_accessory/config/deploy.yml new file mode 100644 index 00000000..bdb547ae --- /dev/null +++ b/test/integration/docker/deployer/app_with_proxied_accessory/config/deploy.yml @@ -0,0 +1,44 @@ +service: app_with_proxied_accessory +image: app_with_proxied_accessory +servers: + - vm1 +env: + clear: + CLEAR_TOKEN: 4321 + CLEAR_TAG: "" + HOST_TOKEN: "${HOST_TOKEN}" +asset_path: /usr/share/nginx/html/versions +proxy: + host: 127.0.0.1 +registry: + server: registry:4443 + username: root + password: root +builder: + driver: docker + arch: <%= Kamal::Utils.docker_arch %> + args: + COMMIT_SHA: <%= `git rev-parse HEAD` %> +accessories: + busybox: + service: custom-busybox + image: registry:4443/busybox:1.36.0 + cmd: sh -c 'echo "Starting busybox..."; trap exit term; while true; do sleep 1; done' + roles: + - web + netcat: + service: netcat + image: registry:4443/busybox:1.36.0 + cmd: > + sh -c 'echo "Starting netcat..."; while true; do echo -e "HTTP/1.1 200 OK\r\nContent-Length: 11\r\n\r\nHello Ruby" | nc -l -p 80; done' + roles: + - web + port: 12345:80 + proxy: + host: netcat + ssl: false + healthcheck: + interval: 1 + timeout: 1 + path: "/" + diff --git a/test/integration/docker/deployer/app_with_proxied_accessory/default.conf b/test/integration/docker/deployer/app_with_proxied_accessory/default.conf new file mode 100644 index 00000000..e37a9bc1 --- /dev/null +++ b/test/integration/docker/deployer/app_with_proxied_accessory/default.conf @@ -0,0 +1,17 @@ +server { + listen 80; + listen [::]:80; + server_name localhost; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} diff --git a/test/integration/proxied_accessory_test.rb b/test/integration/proxied_accessory_test.rb new file mode 100644 index 00000000..10f3cff8 --- /dev/null +++ b/test/integration/proxied_accessory_test.rb @@ -0,0 +1,63 @@ +require_relative "integration_test" + +class ProxiedAccessoryTest < IntegrationTest + test "boot, stop, start, restart, logs, remove" do + @app = "app_with_proxied_accessory" + + kamal :deploy + + kamal :accessory, :boot, :netcat + assert_accessory_running :netcat + assert_netcat_is_up + + kamal :accessory, :stop, :netcat + assert_accessory_not_running :netcat + assert_netcat_not_found + + kamal :accessory, :start, :netcat + assert_accessory_running :netcat + assert_netcat_is_up + + kamal :accessory, :restart, :netcat + assert_accessory_running :netcat + assert_netcat_is_up + + kamal :accessory, :remove, :netcat, "-y" + assert_accessory_not_running :netcat + assert_netcat_not_found + end + + private + def assert_accessory_running(name) + assert_match /registry:4443\/busybox:1.36.0 "sh -c 'echo \\"Start/, accessory_details(name) + end + + def assert_accessory_not_running(name) + assert_no_match /registry:4443\/busybox:1.36.0 "sh -c 'echo \\"Start/, accessory_details(name) + end + + def accessory_details(name) + kamal :accessory, :details, name, capture: true + end + + def assert_netcat_is_up + response = netcat_response + debug_response_code(response, "200") + assert_equal "200", response.code + end + + def assert_netcat_not_found + response = netcat_response + debug_response_code(response, "404") + assert_equal "404", response.code + end + + def netcat_response + uri = URI.parse("http://127.0.0.1:12345/up") + http = Net::HTTP.new(uri.host, uri.port) + request = Net::HTTP::Get.new(uri) + request["Host"] = "netcat" + + http.request(request) + end +end