From d6c4411e97a1fedce07f2fc1698a953c0f1fa56d Mon Sep 17 00:00:00 2001 From: Aleksandr Lossenko Date: Sun, 3 Nov 2024 15:27:54 +0100 Subject: [PATCH 01/43] add support to enpass --- lib/kamal/secrets/adapters/enpass.rb | 59 +++++++++++++++++ test/secrets/enpass_adapter_test.rb | 99 ++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 lib/kamal/secrets/adapters/enpass.rb create mode 100644 test/secrets/enpass_adapter_test.rb diff --git a/lib/kamal/secrets/adapters/enpass.rb b/lib/kamal/secrets/adapters/enpass.rb new file mode 100644 index 00000000..b698d8c6 --- /dev/null +++ b/lib/kamal/secrets/adapters/enpass.rb @@ -0,0 +1,59 @@ +require "open3" + +class Kamal::Secrets::Adapters::Enpass < Kamal::Secrets::Adapters::Base + private + def login(account) + # There is no concept of session in enpass-cli + true + end + + def fetch_secrets(secrets, account:, session:) + secrets_titles = fetch_secret_titles(secrets) + + # Enpass outputs result as stderr, I did not find a way to stub backticks and output to stderr. Open3 did the job. + _stdout, stderr, status = Open3.capture3("enpass-cli -vault #{account.shellescape} show #{secrets_titles.map(&:shellescape).join(" ")}") + raise RuntimeError, "Could not read #{secrets} from Enpass" unless status.success? + + parse_result_and_take_secrets(stderr, secrets) + end + + def check_dependencies! + raise RuntimeError, "Enpass CLI is not installed" unless cli_installed? + end + + def cli_installed? + `enpass-cli version 2> /dev/null` + $?.success? + end + + def fetch_secret_titles(secrets) + secrets.reduce(Set.new) do |acc, secret| + # Use rpartition to split the string at the last '/' + key, separator, value = secret.rpartition("/") + if key.empty? + acc << value + else + acc << key + end + end.to_a + end + + def parse_result_and_take_secrets(unparsed_result, secrets) + unparsed_result.split("\n").reduce({}) do |acc, line| + title = line[/title:\s*(\w+)/, 1] + label = line[/label:\s*(.*?)\s{2}/, 1] + password = line[/password:\s*([^"]+)/, 1] + + # If title and label are not empty and password is defined, add to the hash + if title && !password.to_s.empty? + key = label.nil? || label.empty? ? title : "#{title}/#{label}" + if secrets.include?(title) || secrets.include?(key) + raise RuntimeError, "#{key} is present more than once" if acc[key] + acc[key] = password + end + end + + acc + end + end +end diff --git a/test/secrets/enpass_adapter_test.rb b/test/secrets/enpass_adapter_test.rb new file mode 100644 index 00000000..d64bb6f6 --- /dev/null +++ b/test/secrets/enpass_adapter_test.rb @@ -0,0 +1,99 @@ +require "test_helper" + +class EnpassAdapterTest < SecretAdapterTestCase + setup do + `true` # Ensure $? is 0 + end + + test "fetch without CLI installed" do + stub_ticks_with("enpass-cli version 2> /dev/null", succeed: false) + + error = assert_raises RuntimeError do + JSON.parse(shellunescape(run_command("fetch", "mynote"))) + end + + assert_equal "Enpass CLI is not installed", error.message + end + + test "fetch one item" do + stub_ticks_with("enpass-cli version 2> /dev/null") + + stderr_response = <<~RESULT + time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: SECRET_1 password: my-password-1 + RESULT + + Open3.stubs(:capture3).returns([ "", stderr_response, OpenStruct.new(success?: true) ]) + + json = JSON.parse(shellunescape(run_command("fetch", "FooBar/SECRET_1"))) + + expected_json = { "FooBar/SECRET_1" => "my-password-1" } + + assert_equal expected_json, json + end + + test "fetch multiple items" do + stub_ticks_with("enpass-cli version 2> /dev/null") + + stderr_response = <<~RESULT + time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: SECRET_1 password: my-password-1 + time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: SECRET_2 password: my-password-2 + time="2024-11-03T13:34:39+01:00" level=info msg="> title: Hello login: cat.: computer label: SECRET_3 password: my-password-3 + RESULT + + Open3.stubs(:capture3).returns([ "", stderr_response, OpenStruct.new(success?: true) ]) + + json = JSON.parse(shellunescape(run_command("fetch", "FooBar/SECRET_1", "FooBar/SECRET_2"))) + + expected_json = { "FooBar/SECRET_1" => "my-password-1", "FooBar/SECRET_2" => "my-password-2" } + + assert_equal expected_json, json + end + + test "fetch multiple items with from" do + stub_ticks_with("enpass-cli version 2> /dev/null") + + stderr_response = <<~RESULT + time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: SECRET_1 password: my-password-1 + time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: SECRET_2 password: my-password-2 + time="2024-11-03T13:34:39+01:00" level=info msg="> title: Hello login: cat.: computer label: SECRET_3 password: my-password-3 + RESULT + + Open3.stubs(:capture3).returns([ "", stderr_response, OpenStruct.new(success?: true) ]) + + json = JSON.parse(shellunescape(run_command("fetch", "--from", "FooBar", "SECRET_1", "SECRET_2"))) + + expected_json = { "FooBar/SECRET_1" => "my-password-1", "FooBar/SECRET_2" => "my-password-2" } + + assert_equal expected_json, json + end + + test "fetch all with from" do + stub_ticks_with("enpass-cli version 2> /dev/null") + + stderr_response = <<~RESULT + time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: SECRET_1 password: my-password-1 + time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: SECRET_2 password: my-password-2 + time="2024-11-03T13:34:39+01:00" level=info msg="> title: Hello login: cat.: computer label: SECRET_3 password: my-password-3 + time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: password: my-password-3 + RESULT + + Open3.stubs(:capture3).returns([ "", stderr_response, OpenStruct.new(success?: true) ]) + + json = JSON.parse(shellunescape(run_command("fetch", "FooBar"))) + + expected_json = { "FooBar/SECRET_1" => "my-password-1", "FooBar/SECRET_2" => "my-password-2", "FooBar" => "my-password-3" } + + assert_equal expected_json, json + end + + private + def run_command(*command) + stdouted do + Kamal::Cli::Secrets.start \ + [ *command, + "-c", "test/fixtures/deploy_with_accessories.yml", + "--adapter", "enpass", + "--account", "vault-path" ] + end + end +end From 4d09f3c242ed7b1fb400044ecbec2f685b4cdf25 Mon Sep 17 00:00:00 2001 From: Aleksandr Lossenko Date: Sun, 3 Nov 2024 15:34:59 +0100 Subject: [PATCH 02/43] add more docs --- lib/kamal/secrets/adapters/enpass.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/kamal/secrets/adapters/enpass.rb b/lib/kamal/secrets/adapters/enpass.rb index b698d8c6..ae257545 100644 --- a/lib/kamal/secrets/adapters/enpass.rb +++ b/lib/kamal/secrets/adapters/enpass.rb @@ -1,5 +1,9 @@ require "open3" +## +# Enpass is different from most password managers, in a way that it's offline. A path to a vault is treated as account. +# +# Pass it like so: `kamal secrets fetch --adapter enpass --account /Users/YOUR_USERNAME/Library/Containers/in.sinew.Enpass-Desktop/Data/Documents/Vaults/primary --from MY_PROD_SERVER` class Kamal::Secrets::Adapters::Enpass < Kamal::Secrets::Adapters::Base private def login(account) @@ -28,7 +32,7 @@ class Kamal::Secrets::Adapters::Enpass < Kamal::Secrets::Adapters::Base def fetch_secret_titles(secrets) secrets.reduce(Set.new) do |acc, secret| - # Use rpartition to split the string at the last '/' + # Sometimes secrets contain a '/', sometimes not key, separator, value = secret.rpartition("/") if key.empty? acc << value @@ -44,7 +48,6 @@ class Kamal::Secrets::Adapters::Enpass < Kamal::Secrets::Adapters::Base label = line[/label:\s*(.*?)\s{2}/, 1] password = line[/password:\s*([^"]+)/, 1] - # If title and label are not empty and password is defined, add to the hash if title && !password.to_s.empty? key = label.nil? || label.empty? ? title : "#{title}/#{label}" if secrets.include?(title) || secrets.include?(key) From b356b08069a312c1a0fec1a9999bcd9c438b61f4 Mon Sep 17 00:00:00 2001 From: Aleksandr Lossenko Date: Sun, 3 Nov 2024 15:50:11 +0100 Subject: [PATCH 03/43] improve password parsing --- lib/kamal/secrets/adapters/enpass.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/kamal/secrets/adapters/enpass.rb b/lib/kamal/secrets/adapters/enpass.rb index ae257545..61636032 100644 --- a/lib/kamal/secrets/adapters/enpass.rb +++ b/lib/kamal/secrets/adapters/enpass.rb @@ -44,9 +44,9 @@ class Kamal::Secrets::Adapters::Enpass < Kamal::Secrets::Adapters::Base def parse_result_and_take_secrets(unparsed_result, secrets) unparsed_result.split("\n").reduce({}) do |acc, line| - title = line[/title:\s*(\w+)/, 1] - label = line[/label:\s*(.*?)\s{2}/, 1] - password = line[/password:\s*([^"]+)/, 1] + title = line[/title:\s{1}(\w+)/, 1] + label = line[/label:\s{1}(.*?)\s{2}/, 1] + password = line[/password:\s{1}([^"]+)/, 1] if title && !password.to_s.empty? key = label.nil? || label.empty? ? title : "#{title}/#{label}" From 8d7a6403b5b2355e57e6f94f915f9f508043c6f1 Mon Sep 17 00:00:00 2001 From: Aleksandr Lossenko Date: Thu, 7 Nov 2024 17:44:28 +0100 Subject: [PATCH 04/43] enpass-cli now has JSON support --- lib/kamal/secrets/adapters/enpass.rb | 35 ++++++++-------- test/secrets/enpass_adapter_test.rb | 60 +++++++++++++++------------- 2 files changed, 52 insertions(+), 43 deletions(-) diff --git a/lib/kamal/secrets/adapters/enpass.rb b/lib/kamal/secrets/adapters/enpass.rb index 61636032..773d1418 100644 --- a/lib/kamal/secrets/adapters/enpass.rb +++ b/lib/kamal/secrets/adapters/enpass.rb @@ -15,10 +15,9 @@ class Kamal::Secrets::Adapters::Enpass < Kamal::Secrets::Adapters::Base secrets_titles = fetch_secret_titles(secrets) # Enpass outputs result as stderr, I did not find a way to stub backticks and output to stderr. Open3 did the job. - _stdout, stderr, status = Open3.capture3("enpass-cli -vault #{account.shellescape} show #{secrets_titles.map(&:shellescape).join(" ")}") - raise RuntimeError, "Could not read #{secrets} from Enpass" unless status.success? + result = `enpass-cli -json -vault #{account.shellescape} show #{secrets.map(&:shellescape).join(" ")}`.strip - parse_result_and_take_secrets(stderr, secrets) + parse_result_and_take_secrets(result, secrets) end def check_dependencies! @@ -31,32 +30,36 @@ class Kamal::Secrets::Adapters::Enpass < Kamal::Secrets::Adapters::Base end def fetch_secret_titles(secrets) - secrets.reduce(Set.new) do |acc, secret| - # Sometimes secrets contain a '/', sometimes not + secrets.reduce(Set.new) do |secret_titles, secret| + # Sometimes secrets contain a '/', when the intent is to fetch a single password for an item. Example: FooBar/DB_PASSWORD + # Another case is, when the intent is to fetch all passwords for an item. Example: FooBar (and FooBar may have multiple different passwords) key, separator, value = secret.rpartition("/") if key.empty? - acc << value + secret_titles << value else - acc << key + secret_titles << key end end.to_a end def parse_result_and_take_secrets(unparsed_result, secrets) - unparsed_result.split("\n").reduce({}) do |acc, line| - title = line[/title:\s{1}(\w+)/, 1] - label = line[/label:\s{1}(.*?)\s{2}/, 1] - password = line[/password:\s{1}([^"]+)/, 1] + result = JSON.parse(unparsed_result) + + result.reduce({}) do |secrets_with_passwords, item| + title = item["title"] + label = item["label"] + password = item["password"] + + if title && password.present? + key = [ title, label ].compact.reject(&:empty?).join("/") - if title && !password.to_s.empty? - key = label.nil? || label.empty? ? title : "#{title}/#{label}" if secrets.include?(title) || secrets.include?(key) - raise RuntimeError, "#{key} is present more than once" if acc[key] - acc[key] = password + raise RuntimeError, "#{key} is present more than once" if secrets_with_passwords[key] + secrets_with_passwords[key] = password end end - acc + secrets_with_passwords end end end diff --git a/test/secrets/enpass_adapter_test.rb b/test/secrets/enpass_adapter_test.rb index d64bb6f6..852c7c9b 100644 --- a/test/secrets/enpass_adapter_test.rb +++ b/test/secrets/enpass_adapter_test.rb @@ -18,11 +18,11 @@ class EnpassAdapterTest < SecretAdapterTestCase test "fetch one item" do stub_ticks_with("enpass-cli version 2> /dev/null") - stderr_response = <<~RESULT - time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: SECRET_1 password: my-password-1 - RESULT - - Open3.stubs(:capture3).returns([ "", stderr_response, OpenStruct.new(success?: true) ]) + stub_ticks + .with("enpass-cli -json -vault vault-path show FooBar/SECRET_1") + .returns(<<~JSON) + [{"category":"computer","label":"SECRET_1","login":"","password":"my-password-1","title":"FooBar","type":"password"}] + JSON json = JSON.parse(shellunescape(run_command("fetch", "FooBar/SECRET_1"))) @@ -34,13 +34,15 @@ class EnpassAdapterTest < SecretAdapterTestCase test "fetch multiple items" do stub_ticks_with("enpass-cli version 2> /dev/null") - stderr_response = <<~RESULT - time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: SECRET_1 password: my-password-1 - time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: SECRET_2 password: my-password-2 - time="2024-11-03T13:34:39+01:00" level=info msg="> title: Hello login: cat.: computer label: SECRET_3 password: my-password-3 - RESULT - - Open3.stubs(:capture3).returns([ "", stderr_response, OpenStruct.new(success?: true) ]) + stub_ticks + .with("enpass-cli -json -vault vault-path show FooBar/SECRET_1 FooBar/SECRET_2") + .returns(<<~JSON) + [ + {"category":"computer","label":"SECRET_1","login":"","password":"my-password-1","title":"FooBar","type":"password"}, + {"category":"computer","label":"SECRET_2","login":"","password":"my-password-2","title":"FooBar","type":"password"}, + {"category":"computer","label":"SECRET_3","login":"","password":"my-password-1","title":"Hello","type":"password"} + ] + JSON json = JSON.parse(shellunescape(run_command("fetch", "FooBar/SECRET_1", "FooBar/SECRET_2"))) @@ -52,13 +54,15 @@ class EnpassAdapterTest < SecretAdapterTestCase test "fetch multiple items with from" do stub_ticks_with("enpass-cli version 2> /dev/null") - stderr_response = <<~RESULT - time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: SECRET_1 password: my-password-1 - time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: SECRET_2 password: my-password-2 - time="2024-11-03T13:34:39+01:00" level=info msg="> title: Hello login: cat.: computer label: SECRET_3 password: my-password-3 - RESULT - - Open3.stubs(:capture3).returns([ "", stderr_response, OpenStruct.new(success?: true) ]) + stub_ticks + .with("enpass-cli -json -vault vault-path show FooBar/SECRET_1 FooBar/SECRET_2") + .returns(<<~JSON) + [ + {"category":"computer","label":"SECRET_1","login":"","password":"my-password-1","title":"FooBar","type":"password"}, + {"category":"computer","label":"SECRET_2","login":"","password":"my-password-2","title":"FooBar","type":"password"}, + {"category":"computer","label":"SECRET_3","login":"","password":"my-password-1","title":"Hello","type":"password"} + ] + JSON json = JSON.parse(shellunescape(run_command("fetch", "--from", "FooBar", "SECRET_1", "SECRET_2"))) @@ -70,14 +74,16 @@ class EnpassAdapterTest < SecretAdapterTestCase test "fetch all with from" do stub_ticks_with("enpass-cli version 2> /dev/null") - stderr_response = <<~RESULT - time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: SECRET_1 password: my-password-1 - time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: SECRET_2 password: my-password-2 - time="2024-11-03T13:34:39+01:00" level=info msg="> title: Hello login: cat.: computer label: SECRET_3 password: my-password-3 - time="2024-11-03T13:34:39+01:00" level=info msg="> title: FooBar login: cat.: computer label: password: my-password-3 - RESULT - - Open3.stubs(:capture3).returns([ "", stderr_response, OpenStruct.new(success?: true) ]) + stub_ticks + .with("enpass-cli -json -vault vault-path show FooBar") + .returns(<<~JSON) + [ + {"category":"computer","label":"SECRET_1","login":"","password":"my-password-1","title":"FooBar","type":"password"}, + {"category":"computer","label":"SECRET_2","login":"","password":"my-password-2","title":"FooBar","type":"password"}, + {"category":"computer","label":"SECRET_3","login":"","password":"my-password-1","title":"Hello","type":"password"}, + {"category":"computer","label":"","login":"","password":"my-password-3","title":"FooBar","type":"password"} + ] + JSON json = JSON.parse(shellunescape(run_command("fetch", "FooBar"))) From c9dec8c79a40ddb53a49de7135e440b481ac4dfe Mon Sep 17 00:00:00 2001 From: Aleksandr Lossenko Date: Thu, 7 Nov 2024 17:45:24 +0100 Subject: [PATCH 05/43] no need for open3 anymore --- lib/kamal/secrets/adapters/enpass.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/kamal/secrets/adapters/enpass.rb b/lib/kamal/secrets/adapters/enpass.rb index 773d1418..8f11536d 100644 --- a/lib/kamal/secrets/adapters/enpass.rb +++ b/lib/kamal/secrets/adapters/enpass.rb @@ -1,5 +1,3 @@ -require "open3" - ## # Enpass is different from most password managers, in a way that it's offline. A path to a vault is treated as account. # From 79bc7584ca36dd1b1af4110574c3c06f653260f5 Mon Sep 17 00:00:00 2001 From: Aleksandr Lossenko Date: Thu, 7 Nov 2024 18:13:37 +0100 Subject: [PATCH 06/43] make --account optional and pass Enpass vault in --from --- lib/kamal/cli/secrets.rb | 2 +- lib/kamal/secrets/adapters/enpass.rb | 25 +++++++++++++---------- test/secrets/enpass_adapter_test.rb | 30 +++------------------------- 3 files changed, 19 insertions(+), 38 deletions(-) diff --git a/lib/kamal/cli/secrets.rb b/lib/kamal/cli/secrets.rb index b094be46..0b458f13 100644 --- a/lib/kamal/cli/secrets.rb +++ b/lib/kamal/cli/secrets.rb @@ -1,7 +1,7 @@ class Kamal::Cli::Secrets < Kamal::Cli::Base desc "fetch [SECRETS...]", "Fetch secrets from a vault" option :adapter, type: :string, aliases: "-a", required: true, desc: "Which vault adapter to use" - option :account, type: :string, required: true, desc: "The account identifier or username" + option :account, type: :string, required: false, desc: "The account identifier or username" option :from, type: :string, required: false, desc: "A vault or folder to fetch the secrets from" option :inline, type: :boolean, required: false, hidden: true def fetch(*secrets) diff --git a/lib/kamal/secrets/adapters/enpass.rb b/lib/kamal/secrets/adapters/enpass.rb index 8f11536d..68fe41a9 100644 --- a/lib/kamal/secrets/adapters/enpass.rb +++ b/lib/kamal/secrets/adapters/enpass.rb @@ -1,19 +1,24 @@ ## -# Enpass is different from most password managers, in a way that it's offline. A path to a vault is treated as account. +# Enpass is different from most password managers, in a way that it's offline and doesn't need an account. # -# Pass it like so: `kamal secrets fetch --adapter enpass --account /Users/YOUR_USERNAME/Library/Containers/in.sinew.Enpass-Desktop/Data/Documents/Vaults/primary --from MY_PROD_SERVER` +# Usage +# +# Fetch all password from FooBar item +# `kamal secrets fetch --adapter enpass --from /Users/YOUR_USERNAME/Library/Containers/in.sinew.Enpass-Desktop/Data/Documents/Vaults/primary FooBar` +# +# Fetch only DB_PASSWORD from FooBar item +# `kamal secrets fetch --adapter enpass --from /Users/YOUR_USERNAME/Library/Containers/in.sinew.Enpass-Desktop/Data/Documents/Vaults/primary FooBar/DB_PASSWORD` class Kamal::Secrets::Adapters::Enpass < Kamal::Secrets::Adapters::Base - private - def login(account) - # There is no concept of session in enpass-cli - true - end + def fetch(secrets, account: nil, from:) + check_dependencies! + fetch_secrets(secrets, from) + end - def fetch_secrets(secrets, account:, session:) + private + def fetch_secrets(secrets, vault) secrets_titles = fetch_secret_titles(secrets) - # Enpass outputs result as stderr, I did not find a way to stub backticks and output to stderr. Open3 did the job. - result = `enpass-cli -json -vault #{account.shellescape} show #{secrets.map(&:shellescape).join(" ")}`.strip + result = `enpass-cli -json -vault #{vault.shellescape} show #{secrets_titles.map(&:shellescape).join(" ")}`.strip parse_result_and_take_secrets(result, secrets) end diff --git a/test/secrets/enpass_adapter_test.rb b/test/secrets/enpass_adapter_test.rb index 852c7c9b..edc49613 100644 --- a/test/secrets/enpass_adapter_test.rb +++ b/test/secrets/enpass_adapter_test.rb @@ -1,10 +1,6 @@ require "test_helper" class EnpassAdapterTest < SecretAdapterTestCase - setup do - `true` # Ensure $? is 0 - end - test "fetch without CLI installed" do stub_ticks_with("enpass-cli version 2> /dev/null", succeed: false) @@ -19,7 +15,7 @@ class EnpassAdapterTest < SecretAdapterTestCase stub_ticks_with("enpass-cli version 2> /dev/null") stub_ticks - .with("enpass-cli -json -vault vault-path show FooBar/SECRET_1") + .with("enpass-cli -json -vault vault-path show FooBar") .returns(<<~JSON) [{"category":"computer","label":"SECRET_1","login":"","password":"my-password-1","title":"FooBar","type":"password"}] JSON @@ -35,7 +31,7 @@ class EnpassAdapterTest < SecretAdapterTestCase stub_ticks_with("enpass-cli version 2> /dev/null") stub_ticks - .with("enpass-cli -json -vault vault-path show FooBar/SECRET_1 FooBar/SECRET_2") + .with("enpass-cli -json -vault vault-path show FooBar") .returns(<<~JSON) [ {"category":"computer","label":"SECRET_1","login":"","password":"my-password-1","title":"FooBar","type":"password"}, @@ -51,26 +47,6 @@ class EnpassAdapterTest < SecretAdapterTestCase assert_equal expected_json, json end - test "fetch multiple items with from" do - stub_ticks_with("enpass-cli version 2> /dev/null") - - stub_ticks - .with("enpass-cli -json -vault vault-path show FooBar/SECRET_1 FooBar/SECRET_2") - .returns(<<~JSON) - [ - {"category":"computer","label":"SECRET_1","login":"","password":"my-password-1","title":"FooBar","type":"password"}, - {"category":"computer","label":"SECRET_2","login":"","password":"my-password-2","title":"FooBar","type":"password"}, - {"category":"computer","label":"SECRET_3","login":"","password":"my-password-1","title":"Hello","type":"password"} - ] - JSON - - json = JSON.parse(shellunescape(run_command("fetch", "--from", "FooBar", "SECRET_1", "SECRET_2"))) - - expected_json = { "FooBar/SECRET_1" => "my-password-1", "FooBar/SECRET_2" => "my-password-2" } - - assert_equal expected_json, json - end - test "fetch all with from" do stub_ticks_with("enpass-cli version 2> /dev/null") @@ -99,7 +75,7 @@ class EnpassAdapterTest < SecretAdapterTestCase [ *command, "-c", "test/fixtures/deploy_with_accessories.yml", "--adapter", "enpass", - "--account", "vault-path" ] + "--from", "vault-path" ] end end end From 3793bdc2c30193dc2272238f253bc528255a72cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Laszlo?= Date: Tue, 19 Nov 2024 22:59:19 +0100 Subject: [PATCH 07/43] Add GCP Secret Manager adapter --- lib/kamal/secrets/adapters.rb | 1 + .../secrets/adapters/gcp_secret_manager.rb | 125 +++++++++++ .../gcp_secret_manager_adapter_test.rb | 211 ++++++++++++++++++ 3 files changed, 337 insertions(+) create mode 100644 lib/kamal/secrets/adapters/gcp_secret_manager.rb create mode 100644 test/secrets/gcp_secret_manager_adapter_test.rb diff --git a/lib/kamal/secrets/adapters.rb b/lib/kamal/secrets/adapters.rb index 439c7208..2abf29ad 100644 --- a/lib/kamal/secrets/adapters.rb +++ b/lib/kamal/secrets/adapters.rb @@ -3,6 +3,7 @@ module Kamal::Secrets::Adapters def self.lookup(name) name = "one_password" if name.downcase == "1password" name = "last_pass" if name.downcase == "lastpass" + name = "gcp_secret_manager" if %w[gcp secret_manager].include? name.downcase adapter_class(name) end diff --git a/lib/kamal/secrets/adapters/gcp_secret_manager.rb b/lib/kamal/secrets/adapters/gcp_secret_manager.rb new file mode 100644 index 00000000..c6d6387b --- /dev/null +++ b/lib/kamal/secrets/adapters/gcp_secret_manager.rb @@ -0,0 +1,125 @@ +class Kamal::Secrets::Adapters::GcpSecretManager < Kamal::Secrets::Adapters::Base + private + def login(account) + # Since only the account option is passed from the cli, we'll use it for: + # - Account + # - GCP project + # - Service account impersonation + # + # Syntax: + # ACCOUNT: USER | USER "," DELEGATION_CHAIN + # USER: DEFAULT_USER | EMAIL + # DELEGATION_CHAIN: EMAIL | EMAIL "," DELEGATION_CHAIN + # EMAIL: + # DEFAULT_USER: "default" + # + # Some valid examples: + # - "my-user@example.com" sets the user + # - "my-user@example.com,my-service-user@example.com" will use my-user and enable service account impersonation as my-service-user + # - "default" will use the default user and no impersonation + # - "default,my-service-user@example.com" will use the default user, and enable service account impersonation as my-service-user + # - "default,my-service-user@example.com,another-service-user@example.com" same as above, but with an impersonation delegation chain + + if !logged_in? + raise RuntimeError, "gcloud is not authenticated, please run `gcloud auth login`" + end + + user, impersonate_service_account = parse_account(account) + + { + user: user, + impersonate_service_account: impersonate_service_account + } + end + + def fetch_secrets(secrets, account:, session:) + # puts("secrets spec: #{secrets.inspect}") + {}.tap do |results| + secrets_with_metadata(secrets).each do |secret, metadata| + project, secret_name, secret_version = metadata + item_name = project == "default" ? secret_name : "#{project}/#{secret_name}" + results[item_name] = fetch_secret(session, project, secret_name, secret_version) + raise RuntimeError, "Could not read #{item_name} from Google Secret Manager" unless $?.success? + end + end + end + + def fetch_secret(session, project, secret_name, secret_version) + secret = run_command("secrets versions access #{secret_version} --secret=#{secret_name.shellescape}", session: session, project: project) + Base64.decode64(secret.dig("payload", "data")) + end + + # The secret needs to at least contain a secret name, but project name, and secret version can also be specified. + # + # The string "default" can be used to refer to the default project configured for gcloud. + # + # The version can be either the string "latest", or a version number. + # + # The following formats are valid: + # + # - The following are all equivalent, and sets project: default, secret name: my-secret, version: latest + # - "my-secret" + # - "default/my-secret" + # - "default/my-secret/latest" + # - "my-secret/latest" in combination with --from=default + # - "my-secret/123" (only in combination with --from=some-project) -> project: some-project, secret name: my-secret, version: 123 + # - "some-project/my-secret/123" -> project: some-project, secret name: my-secret, version: 123 + def secrets_with_metadata(secrets) + {}.tap do |items| + secrets.each do |secret| + parts = secret.split("/") + parts.unshift("default") if parts.length == 1 + project = parts.shift + secret_name = parts.shift + secret_version = parts.shift || "latest" + + items[secret] = [ project, secret_name, secret_version ] + end + end + end + + def run_command(command, session: nil, project: nil) + full_command = [ "gcloud", command ] + full_command << "--project=#{project}" unless project == "default" + full_command << "--account=#{session[:user]}" unless session[:user] == "default" + full_command << "--impersonate-service-account=#{session[:impersonate_service_account]}" if session[:impersonate_service_account] + full_command << "--format=json" + full_command = full_command.join(" ") + + result = `#{full_command}`.strip + JSON.parse(result) + end + + def check_dependencies! + raise RuntimeError, "gcloud CLI is not installed" unless cli_installed? + end + + def cli_installed? + `gcloud --version 2> /dev/null` + $?.success? + end + + def logged_in? + JSON.parse(`gcloud auth list --format=json`).any? + end + + def parse_account(account) + return "default", nil if account == "default" + + parts = account.split(",", 2) + + if parts.length == 2 + return parts.shift, parts.shift + elsif parts.length != 1 + raise RuntimeError, "Invalid account, too many parts: #{account}" + elsif is_user?(account) + return account, nil + end + + raise RuntimeError, "Invalid account, not a user: #{account}" + end + + def is_user?(candidate) + candidate.include?("@") + end +end diff --git a/test/secrets/gcp_secret_manager_adapter_test.rb b/test/secrets/gcp_secret_manager_adapter_test.rb new file mode 100644 index 00000000..3369d08e --- /dev/null +++ b/test/secrets/gcp_secret_manager_adapter_test.rb @@ -0,0 +1,211 @@ +require "test_helper" + +class GcpSecretManagerAdapterTest < SecretAdapterTestCase + test "fetch" do + stub_gcloud_version + stub_authenticated + stub_mypassword + + json = JSON.parse(shellunescape(run_command("fetch", "mypassword"))) + + expected_json = { "mypassword"=>"secret123" } + + assert_equal expected_json, json + end + + test "fetch unauthenticated" do + stub_ticks.with("gcloud --version 2> /dev/null") + + stub_mypassword + stub_unauthenticated + + error = assert_raises RuntimeError do + JSON.parse(shellunescape(run_command("fetch", "mypassword"))) + end + + assert_match(/not authenticated/, error.message) + end + + test "fetch with from" do + stub_gcloud_version + stub_authenticated + stub_items(0, project: "other-project") + stub_items(1, project: "other-project") + stub_items(2, project: "other-project") + + json = JSON.parse(shellunescape(run_command("fetch", "--from", "other-project", "item1", "item2", "item3"))) + + expected_json = { + "other-project/item1"=>"secret1", "other-project/item2"=>"secret2", "other-project/item3"=>"secret3" + } + + assert_equal expected_json, json + end + + test "fetch with multiple projects" do + stub_gcloud_version + stub_authenticated + stub_items(0, project: "some-project") + stub_items(1, project: "project-confidence") + stub_items(2, project: "manhattan-project") + + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1", "project-confidence/item2", "manhattan-project/item3"))) + + expected_json = { + "some-project/item1"=>"secret1", "project-confidence/item2"=>"secret2", "manhattan-project/item3"=>"secret3" + } + + assert_equal expected_json, json + end + + test "fetch with specific version" do + stub_gcloud_version + stub_authenticated + stub_items(0, project: "some-project", version: "123") + + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123"))) + + expected_json = { + "some-project/item1"=>"secret1" + } + + assert_equal expected_json, json + end + + test "fetch with non-default account" do + stub_gcloud_version + stub_authenticated + stub_items(0, project: "some-project", version: "123", account: "email@example.com") + + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123", account: "email@example.com"))) + + expected_json = { + "some-project/item1"=>"secret1" + } + + assert_equal expected_json, json + end + + test "fetch with service account impersonation" do + stub_gcloud_version + stub_authenticated + stub_items(0, project: "some-project", version: "123", impersonate_service_account: "service-user@example.com") + + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123", account: "default,service-user@example.com"))) + + expected_json = { + "some-project/item1"=>"secret1" + } + + assert_equal expected_json, json + end + + test "fetch with delegation chain and specific user" do + stub_gcloud_version + stub_authenticated + stub_items(0, project: "some-project", version: "123", account: "user@example.com", impersonate_service_account: "service-user@example.com,service-user2@example.com") + + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123", account: "user@example.com,service-user@example.com,service-user2@example.com"))) + + expected_json = { + "some-project/item1"=>"secret1" + } + + assert_equal expected_json, json + end + + test "fetch with non-default account and service account impersonation" do + stub_gcloud_version + stub_authenticated + stub_items(0, project: "some-project", version: "123", account: "email@example.com", impersonate_service_account: "service-user@example.com") + + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123", account: "email@example.com,service-user@example.com"))) + + expected_json = { + "some-project/item1"=>"secret1" + } + + assert_equal expected_json, json + end + + test "fetch without CLI installed" do + stub_gcloud_version(succeed: false) + + error = assert_raises RuntimeError do + JSON.parse(shellunescape(run_command("fetch", "item1"))) + end + assert_equal "gcloud CLI is not installed", error.message + end + + private + def run_command(*command, account: "default") + stdouted do + Kamal::Cli::Secrets.start \ + [ *command, + "-c", "test/fixtures/deploy_with_accessories.yml", + "--adapter", "gcp_secret_manager", + "--account", account ] + end + end + + def stub_gcloud_version(succeed: true) + stub_ticks_with("gcloud --version 2> /dev/null", succeed: succeed) + end + + def stub_authenticated + stub_ticks + .with("gcloud auth list --format=json") + .returns(<<~JSON) + [ + { + "account": "email@example.com", + "status": "ACTIVE" + } + ] + JSON + end + + def stub_unauthenticated + stub_ticks + .with("gcloud auth list --format=json") + .returns("[]") + end + + def stub_mypassword + stub_ticks + .with("gcloud secrets versions access latest --secret=mypassword --format=json") + .returns(<<~JSON) + { + "name": "projects/000000000/secrets/mypassword/versions/1", + "payload": { + "data": "c2VjcmV0MTIz", + "dataCrc32c": "2522602764" + } + } + JSON + end + + def stub_items(n, project: nil, account: nil, version: "latest", impersonate_service_account: nil) + payloads = [ + { data: "c2VjcmV0MQ==", checksum: 1846998209 }, + { data: "c2VjcmV0Mg==", checksum: 2101741365 }, + { data: "c2VjcmV0Mw==", checksum: 2402124854 } + ] + stub_ticks + .with("gcloud secrets versions access #{version} " \ + "--secret=item#{n + 1}" \ + "#{" --project=#{project}" if project}" \ + "#{" --account=#{account}" if account}" \ + "#{" --impersonate-service-account=#{impersonate_service_account}" if impersonate_service_account} " \ + "--format=json") + .returns(<<~JSON) + { + "name": "projects/000000001/secrets/item1/versions/1", + "payload": { + "data": "#{payloads[n][:data]}", + "dataCrc32c": "#{payloads[n][:checksum]}" + } + } + JSON + end +end From a07ef64fade353aec99a4b2c6715091a55e6991f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Laszlo?= Date: Wed, 20 Nov 2024 15:27:51 +0100 Subject: [PATCH 08/43] Fix --account documentation --- lib/kamal/secrets/adapters/gcp_secret_manager.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/kamal/secrets/adapters/gcp_secret_manager.rb b/lib/kamal/secrets/adapters/gcp_secret_manager.rb index c6d6387b..25c9fa00 100644 --- a/lib/kamal/secrets/adapters/gcp_secret_manager.rb +++ b/lib/kamal/secrets/adapters/gcp_secret_manager.rb @@ -1,10 +1,8 @@ class Kamal::Secrets::Adapters::GcpSecretManager < Kamal::Secrets::Adapters::Base private def login(account) - # Since only the account option is passed from the cli, we'll use it for: - # - Account - # - GCP project - # - Service account impersonation + # Since only the account option is passed from the cli, we'll use it for both account and service account + # impersonation. # # Syntax: # ACCOUNT: USER | USER "," DELEGATION_CHAIN From 0c9a367efcc11180e5930dc225e56ab3b9bd791f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Laszlo?= Date: Wed, 27 Nov 2024 13:33:04 +0100 Subject: [PATCH 09/43] Remove overly generic 'secret_manager' alias --- lib/kamal/secrets/adapters.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kamal/secrets/adapters.rb b/lib/kamal/secrets/adapters.rb index 2abf29ad..e51c73ef 100644 --- a/lib/kamal/secrets/adapters.rb +++ b/lib/kamal/secrets/adapters.rb @@ -3,7 +3,7 @@ module Kamal::Secrets::Adapters def self.lookup(name) name = "one_password" if name.downcase == "1password" name = "last_pass" if name.downcase == "lastpass" - name = "gcp_secret_manager" if %w[gcp secret_manager].include? name.downcase + name = "gcp_secret_manager" if name.downcase == "gcp" adapter_class(name) end From 11e4f37409beeced2db093bd71fbba0a8a9ceb8a Mon Sep 17 00:00:00 2001 From: Matthew Croall Date: Sat, 30 Nov 2024 11:10:49 +1030 Subject: [PATCH 10/43] Add proxy boot_config --publish-ip argument --- lib/kamal/cli/proxy.rb | 3 ++- lib/kamal/configuration.rb | 5 +++-- test/cli/proxy_test.rb | 9 +++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/kamal/cli/proxy.rb b/lib/kamal/cli/proxy.rb index d0e9ba2b..6b42adad 100644 --- a/lib/kamal/cli/proxy.rb +++ b/lib/kamal/cli/proxy.rb @@ -23,6 +23,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base desc "boot_config ", "Manage kamal-proxy boot configuration" option :publish, type: :boolean, default: true, desc: "Publish the proxy ports on the host" + option :publish_ip, type: :string, desc: "IP address to bind HTTP/HTTPS traffic to. Defaults to all interfaces" option :http_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTP_PORT, desc: "HTTP port to publish on the host" option :https_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTPS_PORT, desc: "HTTPS port to publish on the host" option :log_max_size, type: :string, default: Kamal::Configuration::PROXY_LOG_MAX_SIZE, desc: "Max size of proxy logs" @@ -31,7 +32,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base case subcommand when "set" boot_options = [ - *(KAMAL.config.proxy_publish_args(options[:http_port], options[:https_port]) if options[:publish]), + *(KAMAL.config.proxy_publish_args(options[:http_port], options[:https_port], options[:publish_ip]) if options[:publish]), *(KAMAL.config.proxy_logging_args(options[:log_max_size])), *options[:docker_options].map { |option| "--#{option}" } ] diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index 021e5e49..cafc25d4 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -249,8 +249,9 @@ class Kamal::Configuration env_tags.detect { |t| t.name == name.to_s } end - def proxy_publish_args(http_port, https_port) - argumentize "--publish", [ "#{http_port}:#{PROXY_HTTP_PORT}", "#{https_port}:#{PROXY_HTTPS_PORT}" ] + def proxy_publish_args(http_port, https_port, bind_ip = nil) + bind_ip = bind_ip ? "#{bind_ip}:" : "" + argumentize "--publish", [ "#{bind_ip}#{http_port}:#{PROXY_HTTP_PORT}", "#{bind_ip}#{https_port}:#{PROXY_HTTPS_PORT}" ] end def proxy_logging_args(max_size) diff --git a/test/cli/proxy_test.rb b/test/cli/proxy_test.rb index 0a890451..f499de3b 100644 --- a/test/cli/proxy_test.rb +++ b/test/cli/proxy_test.rb @@ -281,6 +281,15 @@ class CliProxyTest < CliTestCase end end + test "boot_config set custom bind ip" do + run_command("boot_config", "set", "--publish-ip", "127.0.0.1").tap do |output| + %w[ 1.1.1.1 1.1.1.2 ].each do |host| + assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output + assert_match "Uploading \"--publish 127.0.0.1:80:80 --publish 127.0.0.1:443:443 --log-opt max-size=10m\" to .kamal/proxy/options on #{host}", output + end + end + end + test "boot_config set docker options" do run_command("boot_config", "set", "--docker_options", "label=foo=bar", "add_host=thishost:thathost").tap do |output| %w[ 1.1.1.1 1.1.1.2 ].each do |host| From ffe1ac34838f503b131a313083832884dec38a83 Mon Sep 17 00:00:00 2001 From: Matthew Croall Date: Tue, 3 Dec 2024 08:11:19 +1030 Subject: [PATCH 11/43] Refactor proxy_publish_args argument concatenation --- lib/kamal/configuration.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index cafc25d4..5099e9ab 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -250,8 +250,9 @@ class Kamal::Configuration end def proxy_publish_args(http_port, https_port, bind_ip = nil) - bind_ip = bind_ip ? "#{bind_ip}:" : "" - argumentize "--publish", [ "#{bind_ip}#{http_port}:#{PROXY_HTTP_PORT}", "#{bind_ip}#{https_port}:#{PROXY_HTTPS_PORT}" ] + publish_http = [ bind_ip, http_port, PROXY_HTTP_PORT ].compact.join(":") + publish_https = [ bind_ip, https_port, PROXY_HTTPS_PORT ].compact.join(":") + argumentize "--publish", [ publish_http, publish_https ] end def proxy_logging_args(max_size) From 0bafa02e7d9266cc274cd3ed03e6b18fcca84a3a Mon Sep 17 00:00:00 2001 From: Matthew Croall Date: Tue, 3 Dec 2024 08:13:20 +1030 Subject: [PATCH 12/43] Rename proxy bind cli argument to publish_host_ip --- lib/kamal/cli/proxy.rb | 4 ++-- test/cli/proxy_test.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/kamal/cli/proxy.rb b/lib/kamal/cli/proxy.rb index 6b42adad..7ff73b4e 100644 --- a/lib/kamal/cli/proxy.rb +++ b/lib/kamal/cli/proxy.rb @@ -23,7 +23,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base desc "boot_config ", "Manage kamal-proxy boot configuration" option :publish, type: :boolean, default: true, desc: "Publish the proxy ports on the host" - option :publish_ip, type: :string, desc: "IP address to bind HTTP/HTTPS traffic to. Defaults to all interfaces" + option :publish_host_ip, type: :string, desc: "Host IP address to bind HTTP/HTTPS traffic to. Defaults to all interfaces" option :http_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTP_PORT, desc: "HTTP port to publish on the host" option :https_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTPS_PORT, desc: "HTTPS port to publish on the host" option :log_max_size, type: :string, default: Kamal::Configuration::PROXY_LOG_MAX_SIZE, desc: "Max size of proxy logs" @@ -32,7 +32,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base case subcommand when "set" boot_options = [ - *(KAMAL.config.proxy_publish_args(options[:http_port], options[:https_port], options[:publish_ip]) if options[:publish]), + *(KAMAL.config.proxy_publish_args(options[:http_port], options[:https_port], options[:publish_host_ip]) if options[:publish]), *(KAMAL.config.proxy_logging_args(options[:log_max_size])), *options[:docker_options].map { |option| "--#{option}" } ] diff --git a/test/cli/proxy_test.rb b/test/cli/proxy_test.rb index f499de3b..961256af 100644 --- a/test/cli/proxy_test.rb +++ b/test/cli/proxy_test.rb @@ -282,7 +282,7 @@ class CliProxyTest < CliTestCase end test "boot_config set custom bind ip" do - run_command("boot_config", "set", "--publish-ip", "127.0.0.1").tap do |output| + run_command("boot_config", "set", "--publish-host-ip", "127.0.0.1").tap do |output| %w[ 1.1.1.1 1.1.1.2 ].each do |host| assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output assert_match "Uploading \"--publish 127.0.0.1:80:80 --publish 127.0.0.1:443:443 --log-opt max-size=10m\" to .kamal/proxy/options on #{host}", output From aa9fe4c525cbdad5c7e08fbec0ffcf27573c8913 Mon Sep 17 00:00:00 2001 From: Omid Andalib <24489388+oandalib@users.noreply.github.com> Date: Sat, 2 Nov 2024 13:51:07 -0700 Subject: [PATCH 13/43] feat: add Bitwarden Secrets Manager adapter --- lib/kamal/secrets/adapters.rb | 1 + .../adapters/bitwarden_secrets_manager.rb | 67 ++++++++++ .../bitwarden_secrets_manager_adapter_test.rb | 119 ++++++++++++++++++ 3 files changed, 187 insertions(+) create mode 100644 lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb create mode 100644 test/secrets/bitwarden_secrets_manager_adapter_test.rb diff --git a/lib/kamal/secrets/adapters.rb b/lib/kamal/secrets/adapters.rb index 439c7208..19e6daed 100644 --- a/lib/kamal/secrets/adapters.rb +++ b/lib/kamal/secrets/adapters.rb @@ -3,6 +3,7 @@ module Kamal::Secrets::Adapters def self.lookup(name) name = "one_password" if name.downcase == "1password" name = "last_pass" if name.downcase == "lastpass" + name = "bitwarden_secrets_manager" if name.downcase == "bitwarden-sm" adapter_class(name) end diff --git a/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb b/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb new file mode 100644 index 00000000..f0a19caa --- /dev/null +++ b/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb @@ -0,0 +1,67 @@ +class Kamal::Secrets::Adapters::BitwardenSecretsManager < Kamal::Secrets::Adapters::Base + def requires_account? + false + end + + private + LIST_ALL_SELECTOR = "all" + LIST_ALL_FROM_PROJECT_SUFFIX = "/all" + LIST_COMMAND = "secret list -o env" + GET_COMMAND = "secret get -o env" + + def fetch_secrets(secrets, account:, session:) + raise RuntimeError, "You must specify what to retrieve from Bitwarden Secrets Manager" if secrets.length == 0 + + if secrets.length == 1 + if secrets[0] == LIST_ALL_SELECTOR + command = LIST_COMMAND + elsif secrets[0].end_with?(LIST_ALL_FROM_PROJECT_SUFFIX) + project = secrets[0].split(LIST_ALL_FROM_PROJECT_SUFFIX).first + command = "#{LIST_COMMAND} #{project}" + end + end + + {}.tap do |results| + if command.nil? + secrets.each do |secret_uuid| + secret = run_command("#{GET_COMMAND} #{secret_uuid}") + raise RuntimeError, "Could not read #{secret_uuid} from Bitwarden Secrets Manager" unless $?.success? + key, value = parse_secret(secret) + results[key] = value + end + else + secrets = run_command(command) + raise RuntimeError, "Could not read secrets from Bitwarden Secrets Manager" unless $?.success? + secrets.split("\n").each do |secret| + key, value = parse_secret(secret) + results[key] = value + end + end + end + end + + def parse_secret(secret) + key, value = secret.split("=", 2) + value = value.gsub(/^"|"$/, "") + [ key, value ] + end + + def run_command(command, session: nil) + full_command = [ "bws", command ].join(" ") + `#{full_command}` + end + + def login(account) + run_command("run 'echo OK'") + raise RuntimeError, "Could not authenticate to Bitwarden Secrets Manager. Did you set a valid access token?" unless $?.success? + end + + def check_dependencies! + raise RuntimeError, "Bitwarden Secrets Manager CLI is not installed" unless cli_installed? + end + + def cli_installed? + `bws --version 2> /dev/null` + $?.success? + end +end diff --git a/test/secrets/bitwarden_secrets_manager_adapter_test.rb b/test/secrets/bitwarden_secrets_manager_adapter_test.rb new file mode 100644 index 00000000..1723da42 --- /dev/null +++ b/test/secrets/bitwarden_secrets_manager_adapter_test.rb @@ -0,0 +1,119 @@ +require "test_helper" + +class BitwardenSecretsManagerAdapterTest < SecretAdapterTestCase + test "fetch with no parameters" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + + error = assert_raises RuntimeError do + (shellunescape(run_command("fetch"))) + end + assert_equal("You must specify what to retrieve from Bitwarden Secrets Manager", error.message) + end + + test "fetch all" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks + .with("bws secret list -o env") + .returns("KAMAL_REGISTRY_PASSWORD=\"some_password\"\nMY_OTHER_SECRET=\"my=weird\"secret\"") + + expected = '{"KAMAL_REGISTRY_PASSWORD":"some_password","MY_OTHER_SECRET":"my\=weird\"secret"}' + actual = shellunescape(run_command("fetch", "all")) + assert_equal expected, actual + end + + test "fetch all with from" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks + .with("bws secret list -o env 82aeb5bd-6958-4a89-8197-eacab758acce") + .returns("KAMAL_REGISTRY_PASSWORD=\"some_password\"\nMY_OTHER_SECRET=\"my=weird\"secret\"") + + expected = '{"KAMAL_REGISTRY_PASSWORD":"some_password","MY_OTHER_SECRET":"my\=weird\"secret"}' + actual = shellunescape(run_command("fetch", "all", "--from", "82aeb5bd-6958-4a89-8197-eacab758acce")) + assert_equal expected, actual + end + + test "fetch item" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks + .with("bws secret get -o env 82aeb5bd-6958-4a89-8197-eacab758acce") + .returns("KAMAL_REGISTRY_PASSWORD=\"some_password\"") + + expected = '{"KAMAL_REGISTRY_PASSWORD":"some_password"}' + actual = shellunescape(run_command("fetch", "82aeb5bd-6958-4a89-8197-eacab758acce")) + assert_equal expected, actual + end + + test "fetch with multiple items" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks + .with("bws secret get -o env 82aeb5bd-6958-4a89-8197-eacab758acce") + .returns("KAMAL_REGISTRY_PASSWORD=\"some_password\"") + stub_ticks + .with("bws secret get -o env 6f8cdf27-de2b-4c77-a35d-07df8050e332") + .returns("MY_OTHER_SECRET=\"my=weird\"secret\"") + + expected = '{"KAMAL_REGISTRY_PASSWORD":"some_password","MY_OTHER_SECRET":"my\=weird\"secret"}' + actual = shellunescape(run_command("fetch", "82aeb5bd-6958-4a89-8197-eacab758acce", "6f8cdf27-de2b-4c77-a35d-07df8050e332")) + assert_equal expected, actual + end + + test "fetch all empty" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks_with("bws secret list -o env", succeed: false).returns("Error:\n0: Received error message from server") + + error = assert_raises RuntimeError do + (shellunescape(run_command("fetch", "all"))) + end + assert_equal("Could not read secrets from Bitwarden Secrets Manager", error.message) + end + + test "fetch nonexistent item" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks_with("bws secret get -o env 82aeb5bd-6958-4a89-8197-eacab758acce", succeed: false) + .returns("ERROR (RuntimeError): Could not read 82aeb5bd-6958-4a89-8197-eacab758acce from Bitwarden Secrets Manager") + + error = assert_raises RuntimeError do + (shellunescape(run_command("fetch", "82aeb5bd-6958-4a89-8197-eacab758acce"))) + end + assert_equal("Could not read 82aeb5bd-6958-4a89-8197-eacab758acce from Bitwarden Secrets Manager", error.message) + end + + test "fetch with no access token" do + stub_ticks.with("bws --version 2> /dev/null") + stub_ticks_with("bws run 'echo OK'", succeed: false) + + error = assert_raises RuntimeError do + (shellunescape(run_command("fetch", "all"))) + end + assert_equal("Could not authenticate to Bitwarden Secrets Manager. Did you set a valid access token?", error.message) + end + + test "fetch without CLI installed" do + stub_ticks_with("bws --version 2> /dev/null", succeed: false) + + error = assert_raises RuntimeError do + shellunescape(run_command("fetch")) + end + assert_equal "Bitwarden Secrets Manager CLI is not installed", error.message + end + + private + def stub_login + stub_ticks.with("bws run 'echo OK'").returns("OK") + end + + def run_command(*command) + stdouted do + Kamal::Cli::Secrets.start \ + [ *command, + "--adapter", "bitwarden-sm" ] + end + end +end From e597ae6155eb81a21de2a814f7e6d5874a236852 Mon Sep 17 00:00:00 2001 From: Matthew Croall Date: Wed, 4 Dec 2024 10:42:50 +1030 Subject: [PATCH 14/43] Add support for multiple publish ip addresses --- lib/kamal/cli/proxy.rb | 2 +- lib/kamal/configuration.rb | 32 ++++++++++++++++++++++++++++---- test/cli/proxy_test.rb | 19 ++++++++++++++++++- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/lib/kamal/cli/proxy.rb b/lib/kamal/cli/proxy.rb index 7ff73b4e..43444539 100644 --- a/lib/kamal/cli/proxy.rb +++ b/lib/kamal/cli/proxy.rb @@ -23,7 +23,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base desc "boot_config ", "Manage kamal-proxy boot configuration" option :publish, type: :boolean, default: true, desc: "Publish the proxy ports on the host" - option :publish_host_ip, type: :string, desc: "Host IP address to bind HTTP/HTTPS traffic to. Defaults to all interfaces" + option :publish_host_ip, type: :string, repeatable: true, default: nil, desc: "Host IP address to bind HTTP/HTTPS traffic to. Defaults to all interfaces" option :http_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTP_PORT, desc: "HTTP port to publish on the host" option :https_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTPS_PORT, desc: "HTTPS port to publish on the host" option :log_max_size, type: :string, default: Kamal::Configuration::PROXY_LOG_MAX_SIZE, desc: "Max size of proxy logs" diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index 5099e9ab..4955928d 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -249,10 +249,16 @@ class Kamal::Configuration env_tags.detect { |t| t.name == name.to_s } end - def proxy_publish_args(http_port, https_port, bind_ip = nil) - publish_http = [ bind_ip, http_port, PROXY_HTTP_PORT ].compact.join(":") - publish_https = [ bind_ip, https_port, PROXY_HTTPS_PORT ].compact.join(":") - argumentize "--publish", [ publish_http, publish_https ] + def proxy_publish_args(http_port, https_port, bind_ips = nil) + ensure_valid_bind_ips(bind_ips) + + (bind_ips || [ nil ]).map do |bind_ip| + bind_ip = format_bind_ip(bind_ip) + publish_http = [ bind_ip, http_port, PROXY_HTTP_PORT ].compact.join(":") + publish_https = [ bind_ip, https_port, PROXY_HTTPS_PORT ].compact.join(":") + + argumentize "--publish", [ publish_http, publish_https ] + end.join(" ") end def proxy_logging_args(max_size) @@ -346,6 +352,15 @@ class Kamal::Configuration true end + def ensure_valid_bind_ips(bind_ips) + bind_ips.present? && bind_ips.each do |ip| + next if ip =~ Resolv::IPv4::Regex || ip =~ Resolv::IPv6::Regex + raise Kamal::ConfigurationError, "Invalid publish IP address: #{ip}" + end + + true + end + def ensure_retain_containers_valid raise Kamal::ConfigurationError, "Must retain at least 1 container" if retain_containers < 1 @@ -377,6 +392,15 @@ class Kamal::Configuration true end + def format_bind_ip(ip) + # Ensure IPv6 address inside square brackets - e.g. [::1] + if ip =~ Resolv::IPv6::Regex && ip !~ /\[.*\]/ + "[#{ip}]" + else + ip + end + end + def role_names raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort end diff --git a/test/cli/proxy_test.rb b/test/cli/proxy_test.rb index 961256af..381fee55 100644 --- a/test/cli/proxy_test.rb +++ b/test/cli/proxy_test.rb @@ -281,7 +281,7 @@ class CliProxyTest < CliTestCase end end - test "boot_config set custom bind ip" do + test "boot_config set bind IP" do run_command("boot_config", "set", "--publish-host-ip", "127.0.0.1").tap do |output| %w[ 1.1.1.1 1.1.1.2 ].each do |host| assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output @@ -290,6 +290,23 @@ class CliProxyTest < CliTestCase end end + test "boot_config set multiple bind IPs" do + run_command("boot_config", "set", "--publish-host-ip", "127.0.0.1", "--publish-host-ip", "::1").tap do |output| + %w[ 1.1.1.1 1.1.1.2 ].each do |host| + assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output + assert_match "Uploading \"--publish 127.0.0.1:80:80 --publish 127.0.0.1:443:443 --publish [::1]:80:80 --publish [::1]:443:443 --log-opt max-size=10m\" to .kamal/proxy/options on #{host}", output + end + end + end + + test "boot_config set invalid bind IPs" do + exception = assert_raises do + run_command("boot_config", "set", "--publish-host-ip", "1.2.3.invalidIP", "--publish-host-ip", "::1") + end + + assert_includes exception.message, "Invalid publish IP address: 1.2.3.invalidIP" + end + test "boot_config set docker options" do run_command("boot_config", "set", "--docker_options", "label=foo=bar", "add_host=thishost:thathost").tap do |output| %w[ 1.1.1.1 1.1.1.2 ].each do |host| From 1c8a56b8cf5afe7691fe100160c556dfe0263811 Mon Sep 17 00:00:00 2001 From: Matthew Croall Date: Wed, 4 Dec 2024 10:44:16 +1030 Subject: [PATCH 15/43] Change invalid publish ip exception class --- lib/kamal/configuration.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index 4955928d..bbb16324 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -355,7 +355,7 @@ class Kamal::Configuration def ensure_valid_bind_ips(bind_ips) bind_ips.present? && bind_ips.each do |ip| next if ip =~ Resolv::IPv4::Regex || ip =~ Resolv::IPv6::Regex - raise Kamal::ConfigurationError, "Invalid publish IP address: #{ip}" + raise ArgumentError, "Invalid publish IP address: #{ip}" end true From 18f2aae9364638f33abdfddfa6aebe65a5c8b26f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Laszlo?= Date: Fri, 6 Dec 2024 17:15:22 +0100 Subject: [PATCH 16/43] Simplify parsing by changing account separators --- .../secrets/adapters/gcp_secret_manager.rb | 22 +++++-------------- .../gcp_secret_manager_adapter_test.rb | 6 ++--- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/lib/kamal/secrets/adapters/gcp_secret_manager.rb b/lib/kamal/secrets/adapters/gcp_secret_manager.rb index 25c9fa00..82fbb0bf 100644 --- a/lib/kamal/secrets/adapters/gcp_secret_manager.rb +++ b/lib/kamal/secrets/adapters/gcp_secret_manager.rb @@ -5,7 +5,7 @@ class Kamal::Secrets::Adapters::GcpSecretManager < Kamal::Secrets::Adapters::Bas # impersonation. # # Syntax: - # ACCOUNT: USER | USER "," DELEGATION_CHAIN + # ACCOUNT: USER | USER "|" DELEGATION_CHAIN # USER: DEFAULT_USER | EMAIL # DELEGATION_CHAIN: EMAIL | EMAIL "," DELEGATION_CHAIN # EMAIL: @@ -13,10 +13,10 @@ class Kamal::Secrets::Adapters::GcpSecretManager < Kamal::Secrets::Adapters::Bas # # Some valid examples: # - "my-user@example.com" sets the user - # - "my-user@example.com,my-service-user@example.com" will use my-user and enable service account impersonation as my-service-user + # - "my-user@example.com|my-service-user@example.com" will use my-user and enable service account impersonation as my-service-user # - "default" will use the default user and no impersonation - # - "default,my-service-user@example.com" will use the default user, and enable service account impersonation as my-service-user - # - "default,my-service-user@example.com,another-service-user@example.com" same as above, but with an impersonation delegation chain + # - "default|my-service-user@example.com" will use the default user, and enable service account impersonation as my-service-user + # - "default|my-service-user@example.com,another-service-user@example.com" same as above, but with an impersonation delegation chain if !logged_in? raise RuntimeError, "gcloud is not authenticated, please run `gcloud auth login`" @@ -102,19 +102,7 @@ class Kamal::Secrets::Adapters::GcpSecretManager < Kamal::Secrets::Adapters::Bas end def parse_account(account) - return "default", nil if account == "default" - - parts = account.split(",", 2) - - if parts.length == 2 - return parts.shift, parts.shift - elsif parts.length != 1 - raise RuntimeError, "Invalid account, too many parts: #{account}" - elsif is_user?(account) - return account, nil - end - - raise RuntimeError, "Invalid account, not a user: #{account}" + account.split("|", 2) end def is_user?(candidate) diff --git a/test/secrets/gcp_secret_manager_adapter_test.rb b/test/secrets/gcp_secret_manager_adapter_test.rb index 3369d08e..d9b30151 100644 --- a/test/secrets/gcp_secret_manager_adapter_test.rb +++ b/test/secrets/gcp_secret_manager_adapter_test.rb @@ -91,7 +91,7 @@ class GcpSecretManagerAdapterTest < SecretAdapterTestCase stub_authenticated stub_items(0, project: "some-project", version: "123", impersonate_service_account: "service-user@example.com") - json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123", account: "default,service-user@example.com"))) + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123", account: "default|service-user@example.com"))) expected_json = { "some-project/item1"=>"secret1" @@ -105,7 +105,7 @@ class GcpSecretManagerAdapterTest < SecretAdapterTestCase stub_authenticated stub_items(0, project: "some-project", version: "123", account: "user@example.com", impersonate_service_account: "service-user@example.com,service-user2@example.com") - json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123", account: "user@example.com,service-user@example.com,service-user2@example.com"))) + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123", account: "user@example.com|service-user@example.com,service-user2@example.com"))) expected_json = { "some-project/item1"=>"secret1" @@ -119,7 +119,7 @@ class GcpSecretManagerAdapterTest < SecretAdapterTestCase stub_authenticated stub_items(0, project: "some-project", version: "123", account: "email@example.com", impersonate_service_account: "service-user@example.com") - json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123", account: "email@example.com,service-user@example.com"))) + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123", account: "email@example.com|service-user@example.com"))) expected_json = { "some-project/item1"=>"secret1" From ea170fbe5e29da40fb7ccb260ec2ccf27b747155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Laszlo?= Date: Fri, 6 Dec 2024 17:22:03 +0100 Subject: [PATCH 17/43] Run gcloud auth login if user is not authenticated --- lib/kamal/secrets/adapters/gcp_secret_manager.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/kamal/secrets/adapters/gcp_secret_manager.rb b/lib/kamal/secrets/adapters/gcp_secret_manager.rb index 82fbb0bf..9945e4fb 100644 --- a/lib/kamal/secrets/adapters/gcp_secret_manager.rb +++ b/lib/kamal/secrets/adapters/gcp_secret_manager.rb @@ -19,7 +19,8 @@ class Kamal::Secrets::Adapters::GcpSecretManager < Kamal::Secrets::Adapters::Bas # - "default|my-service-user@example.com,another-service-user@example.com" same as above, but with an impersonation delegation chain if !logged_in? - raise RuntimeError, "gcloud is not authenticated, please run `gcloud auth login`" + `gcloud auth login` + raise RuntimeError, "gcloud is not authenticated, please run `gcloud auth login`" if !logged_in? end user, impersonate_service_account = parse_account(account) From dc64aaa0def7c42c7a7b88faa86ba6a4481e8ab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Laszlo?= Date: Fri, 6 Dec 2024 17:32:01 +0100 Subject: [PATCH 18/43] Add gcloud auth login invocation to test --- test/secrets/gcp_secret_manager_adapter_test.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/secrets/gcp_secret_manager_adapter_test.rb b/test/secrets/gcp_secret_manager_adapter_test.rb index d9b30151..5d852df7 100644 --- a/test/secrets/gcp_secret_manager_adapter_test.rb +++ b/test/secrets/gcp_secret_manager_adapter_test.rb @@ -169,6 +169,15 @@ class GcpSecretManagerAdapterTest < SecretAdapterTestCase stub_ticks .with("gcloud auth list --format=json") .returns("[]") + + stub_ticks + .with("gcloud auth login") + .returns(<<~JSON) + { + "expired": false, + "valid": true + } + JSON end def stub_mypassword From 19b4359b17db53b96ffd8483b590fb4a88619917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Laszlo?= Date: Fri, 6 Dec 2024 17:32:31 +0100 Subject: [PATCH 19/43] Use a nil session --- .../secrets/adapters/gcp_secret_manager.rb | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/lib/kamal/secrets/adapters/gcp_secret_manager.rb b/lib/kamal/secrets/adapters/gcp_secret_manager.rb index 9945e4fb..ada16c38 100644 --- a/lib/kamal/secrets/adapters/gcp_secret_manager.rb +++ b/lib/kamal/secrets/adapters/gcp_secret_manager.rb @@ -23,28 +23,29 @@ class Kamal::Secrets::Adapters::GcpSecretManager < Kamal::Secrets::Adapters::Bas raise RuntimeError, "gcloud is not authenticated, please run `gcloud auth login`" if !logged_in? end - user, impersonate_service_account = parse_account(account) - - { - user: user, - impersonate_service_account: impersonate_service_account - } + nil end def fetch_secrets(secrets, account:, session:) - # puts("secrets spec: #{secrets.inspect}") + user, service_account = parse_account(account) + {}.tap do |results| secrets_with_metadata(secrets).each do |secret, metadata| project, secret_name, secret_version = metadata item_name = project == "default" ? secret_name : "#{project}/#{secret_name}" - results[item_name] = fetch_secret(session, project, secret_name, secret_version) + results[item_name] = fetch_secret(project, secret_name, secret_version, user, service_account) raise RuntimeError, "Could not read #{item_name} from Google Secret Manager" unless $?.success? end end end - def fetch_secret(session, project, secret_name, secret_version) - secret = run_command("secrets versions access #{secret_version} --secret=#{secret_name.shellescape}", session: session, project: project) + def fetch_secret(project, secret_name, secret_version, user, service_account) + secret = run_command( + "secrets versions access #{secret_version} --secret=#{secret_name.shellescape}", + project: project, + user: user, + service_account: service_account + ) Base64.decode64(secret.dig("payload", "data")) end @@ -77,11 +78,11 @@ class Kamal::Secrets::Adapters::GcpSecretManager < Kamal::Secrets::Adapters::Bas end end - def run_command(command, session: nil, project: nil) + def run_command(command, project: "default", user: "default", service_account: nil) full_command = [ "gcloud", command ] full_command << "--project=#{project}" unless project == "default" - full_command << "--account=#{session[:user]}" unless session[:user] == "default" - full_command << "--impersonate-service-account=#{session[:impersonate_service_account]}" if session[:impersonate_service_account] + full_command << "--account=#{user}" unless user == "default" + full_command << "--impersonate-service-account=#{service_account}" if service_account full_command << "--format=json" full_command = full_command.join(" ") From eb82b4a753e42037730d68a3c4adac6de1beeca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Laszlo?= Date: Fri, 6 Dec 2024 17:40:08 +0100 Subject: [PATCH 20/43] Keep the 'default' prefix for secret items --- lib/kamal/secrets/adapters/gcp_secret_manager.rb | 5 ++--- test/secrets/gcp_secret_manager_adapter_test.rb | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/kamal/secrets/adapters/gcp_secret_manager.rb b/lib/kamal/secrets/adapters/gcp_secret_manager.rb index ada16c38..f78efbcd 100644 --- a/lib/kamal/secrets/adapters/gcp_secret_manager.rb +++ b/lib/kamal/secrets/adapters/gcp_secret_manager.rb @@ -30,9 +30,8 @@ class Kamal::Secrets::Adapters::GcpSecretManager < Kamal::Secrets::Adapters::Bas user, service_account = parse_account(account) {}.tap do |results| - secrets_with_metadata(secrets).each do |secret, metadata| - project, secret_name, secret_version = metadata - item_name = project == "default" ? secret_name : "#{project}/#{secret_name}" + secrets_with_metadata(secrets).each do |secret, (project, secret_name, secret_version)| + item_name = "#{project}/#{secret_name}" results[item_name] = fetch_secret(project, secret_name, secret_version, user, service_account) raise RuntimeError, "Could not read #{item_name} from Google Secret Manager" unless $?.success? end diff --git a/test/secrets/gcp_secret_manager_adapter_test.rb b/test/secrets/gcp_secret_manager_adapter_test.rb index 5d852df7..341f42e9 100644 --- a/test/secrets/gcp_secret_manager_adapter_test.rb +++ b/test/secrets/gcp_secret_manager_adapter_test.rb @@ -8,7 +8,7 @@ class GcpSecretManagerAdapterTest < SecretAdapterTestCase json = JSON.parse(shellunescape(run_command("fetch", "mypassword"))) - expected_json = { "mypassword"=>"secret123" } + expected_json = { "default/mypassword"=>"secret123" } assert_equal expected_json, json end From 8103d686888a1a3234c968d34258a50fb2df8426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Laszlo?= Date: Fri, 6 Dec 2024 17:43:41 +0100 Subject: [PATCH 21/43] Shellescape all interpolated strings in commands --- lib/kamal/secrets/adapters/gcp_secret_manager.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/kamal/secrets/adapters/gcp_secret_manager.rb b/lib/kamal/secrets/adapters/gcp_secret_manager.rb index f78efbcd..b8dfebf3 100644 --- a/lib/kamal/secrets/adapters/gcp_secret_manager.rb +++ b/lib/kamal/secrets/adapters/gcp_secret_manager.rb @@ -40,7 +40,7 @@ class Kamal::Secrets::Adapters::GcpSecretManager < Kamal::Secrets::Adapters::Bas def fetch_secret(project, secret_name, secret_version, user, service_account) secret = run_command( - "secrets versions access #{secret_version} --secret=#{secret_name.shellescape}", + "secrets versions access #{secret_version.shellescape} --secret=#{secret_name.shellescape}", project: project, user: user, service_account: service_account @@ -79,9 +79,9 @@ class Kamal::Secrets::Adapters::GcpSecretManager < Kamal::Secrets::Adapters::Bas def run_command(command, project: "default", user: "default", service_account: nil) full_command = [ "gcloud", command ] - full_command << "--project=#{project}" unless project == "default" - full_command << "--account=#{user}" unless user == "default" - full_command << "--impersonate-service-account=#{service_account}" if service_account + full_command << "--project=#{project.shellescape}" unless project == "default" + full_command << "--account=#{user.shellescape}" unless user == "default" + full_command << "--impersonate-service-account=#{service_account.shellescape}" if service_account full_command << "--format=json" full_command = full_command.join(" ") From c56edba4a94a82dc38d238b0cfa6117486acab76 Mon Sep 17 00:00:00 2001 From: Piotrek O Date: Wed, 18 Dec 2024 11:35:57 +0100 Subject: [PATCH 22/43] Boot proxy on server setup --- lib/kamal/cli/main.rb | 3 +++ test/cli/main_test.rb | 1 + 2 files changed, 4 insertions(+) diff --git a/lib/kamal/cli/main.rb b/lib/kamal/cli/main.rb index 5fdb5469..b06a3447 100644 --- a/lib/kamal/cli/main.rb +++ b/lib/kamal/cli/main.rb @@ -9,6 +9,9 @@ class Kamal::Cli::Main < Kamal::Cli::Base say "Ensure Docker is installed...", :magenta invoke "kamal:cli:server:bootstrap", [], invoke_options + say "Ensure kamal-proxy is running...", :magenta + invoke "kamal:cli:proxy:boot", [], invoke_options + invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options deploy end diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index cd0efe1f..8a439c49 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -8,6 +8,7 @@ class CliMainTest < CliTestCase invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options) Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options) Kamal::Cli::Main.any_instance.expects(:deploy) From 89db5025a0e2bc3eb10e8bdfc7b2100e19728b2a Mon Sep 17 00:00:00 2001 From: Mike Moore Date: Thu, 19 Dec 2024 09:28:37 -0700 Subject: [PATCH 23/43] Configure Thor to "exit on failure". --- lib/kamal/cli/base.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kamal/cli/base.rb b/lib/kamal/cli/base.rb index 4aebfd90..2d876002 100644 --- a/lib/kamal/cli/base.rb +++ b/lib/kamal/cli/base.rb @@ -5,7 +5,7 @@ module Kamal::Cli class Base < Thor include SSHKit::DSL - def self.exit_on_failure?() false end + def self.exit_on_failure?() true end def self.dynamic_command_class() Kamal::Cli::Alias::Command end class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging" From 39e2c4f848b4a32248f894a8832613e4e0e28e32 Mon Sep 17 00:00:00 2001 From: Mike Moore Date: Thu, 19 Dec 2024 12:14:00 -0700 Subject: [PATCH 24/43] Trying the new method for setting proxy boot config. --- test/integration/main_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb index ce32e640..c72c14dc 100644 --- a/test/integration/main_test.rb +++ b/test/integration/main_test.rb @@ -90,7 +90,7 @@ class MainTest < IntegrationTest test "setup and remove" do @app = "app_with_roles" - kamal :proxy, :set_config, + kamal :proxy, :boot_config, "set", "--publish=false", "--options=label=traefik.http.services.kamal_proxy.loadbalancer.server.scheme=http", "label=traefik.http.routers.kamal_proxy.rule=PathPrefix\\\(\\\`/\\\`\\\)", From 32e1b6504dc82de181b8013e89234b0efdbf4dca Mon Sep 17 00:00:00 2001 From: Mike Moore Date: Fri, 20 Dec 2024 08:26:14 -0700 Subject: [PATCH 25/43] Re-trigger GitHub actions. From 2943c4a3010364dd1987e78a594664f5727d404c Mon Sep 17 00:00:00 2001 From: Mike Moore Date: Fri, 20 Dec 2024 08:45:47 -0700 Subject: [PATCH 26/43] Use the newer option name. --- test/integration/main_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb index c72c14dc..a48051fe 100644 --- a/test/integration/main_test.rb +++ b/test/integration/main_test.rb @@ -92,7 +92,7 @@ class MainTest < IntegrationTest kamal :proxy, :boot_config, "set", "--publish=false", - "--options=label=traefik.http.services.kamal_proxy.loadbalancer.server.scheme=http", + "--docker-options=label=traefik.http.services.kamal_proxy.loadbalancer.server.scheme=http", "label=traefik.http.routers.kamal_proxy.rule=PathPrefix\\\(\\\`/\\\`\\\)", "label=traefik.http.routers.kamal_proxy.priority=2" From b5aee11a40cba8dc7f1a090207a0da8531961d34 Mon Sep 17 00:00:00 2001 From: Pavel Shpak Date: Sun, 22 Dec 2024 02:50:53 +0200 Subject: [PATCH 27/43] [Feature] Add optional accessory registry. Add test cases to cover new option. --- lib/kamal/cli/accessory.rb | 2 +- lib/kamal/commands/accessory.rb | 6 +-- lib/kamal/commands/registry.rb | 16 ++++--- lib/kamal/configuration.rb | 11 +---- lib/kamal/configuration/accessory.rb | 45 ++++++++++------- lib/kamal/configuration/docs/accessory.yml | 20 +++++++- lib/kamal/configuration/registry.rb | 12 ++--- test/cli/accessory_test.rb | 44 ++++++++++------- test/commands/accessory_test.rb | 12 ++--- test/commands/registry_test.rb | 48 +++++++++++++++++-- test/configuration/accessory_test.rb | 22 ++++++++- ..._accessories_with_different_registries.yml | 47 ++++++++++++++++++ 12 files changed, 208 insertions(+), 77 deletions(-) create mode 100644 test/fixtures/deploy_with_accessories_with_different_registries.yml diff --git a/lib/kamal/cli/accessory.rb b/lib/kamal/cli/accessory.rb index 00999b2d..c95cbb1e 100644 --- a/lib/kamal/cli/accessory.rb +++ b/lib/kamal/cli/accessory.rb @@ -292,7 +292,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base def prepare(name) with_accessory(name) do |accessory, hosts| on(hosts) do - execute *KAMAL.registry.login + execute *KAMAL.registry.login(registry_config: accessory.registry) execute *KAMAL.docker.create_network rescue SSHKit::Command::Failed => e raise unless e.message.include?("already exists") diff --git a/lib/kamal/commands/accessory.rb b/lib/kamal/commands/accessory.rb index 281b8713..77ceb607 100644 --- a/lib/kamal/commands/accessory.rb +++ b/lib/kamal/commands/accessory.rb @@ -4,11 +4,10 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base attr_reader :accessory_config delegate :service_name, :image, :hosts, :port, :files, :directories, :cmd, :network_args, :publish_args, :env_args, :volume_args, :label_args, :option_args, - :secrets_io, :secrets_path, :env_directory, :proxy, :running_proxy?, + :secrets_io, :secrets_path, :env_directory, :proxy, :running_proxy?, :registry, to: :accessory_config delegate :proxy_container_name, to: :config - def initialize(config, name:) super(config) @accessory_config = config.accessory(name) @@ -42,7 +41,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base docker :ps, *service_filter end - def logs(timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil) pipe \ docker(:logs, service_name, (" --since #{since}" if since), (" --tail #{lines}" if lines), ("--timestamps" if timestamps), "2>&1"), @@ -56,7 +54,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base (%(grep "#{grep}"#{" #{grep_options}" if grep_options}) if grep) end - def execute_in_existing_container(*command, interactive: false) docker :exec, ("-it" if interactive), @@ -87,7 +84,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base super command, host: hosts.first end - def ensure_local_file_present(local_file) if !local_file.is_a?(StringIO) && !Pathname.new(local_file).exist? raise "Missing file: #{local_file}" diff --git a/lib/kamal/commands/registry.rb b/lib/kamal/commands/registry.rb index 69f95360..b17fbf31 100644 --- a/lib/kamal/commands/registry.rb +++ b/lib/kamal/commands/registry.rb @@ -1,14 +1,16 @@ class Kamal::Commands::Registry < Kamal::Commands::Base - delegate :registry, to: :config + def login(registry_config: nil) + registry_config ||= config.registry - def login docker :login, - registry.server, - "-u", sensitive(Kamal::Utils.escape_shell_value(registry.username)), - "-p", sensitive(Kamal::Utils.escape_shell_value(registry.password)) + registry_config.server, + "-u", sensitive(Kamal::Utils.escape_shell_value(registry_config.username)), + "-p", sensitive(Kamal::Utils.escape_shell_value(registry_config.password)) end - def logout - docker :logout, registry.server + def logout(registry_config: nil) + registry_config ||= config.registry + + docker :logout, registry_config.server end end diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index 52c807d7..827e1204 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -59,7 +59,7 @@ class Kamal::Configuration # Eager load config to validate it, these are first as they have dependencies later on @servers = Servers.new(config: self) - @registry = Registry.new(config: self) + @registry = Registry.new(config: raw_config, secrets: secrets) @accessories = @raw_config.accessories&.keys&.collect { |name| Accessory.new(name, config: self) } || [] @aliases = @raw_config.aliases&.keys&.to_h { |name| [ name, Alias.new(name, config: self) ] } || {} @@ -82,7 +82,6 @@ class Kamal::Configuration ensure_unique_hosts_for_ssl_roles end - def version=(version) @declared_version = version end @@ -106,7 +105,6 @@ class Kamal::Configuration raw_config.minimum_version end - def roles servers.roles end @@ -119,7 +117,6 @@ class Kamal::Configuration accessories.detect { |a| a.name == name.to_s } end - def all_hosts (roles + accessories).flat_map(&:hosts).uniq end @@ -180,7 +177,6 @@ class Kamal::Configuration raw_config.retain_containers || 5 end - def volume_args if raw_config.volumes.present? argumentize "--volume", raw_config.volumes @@ -193,7 +189,6 @@ class Kamal::Configuration logging.args end - def readiness_delay raw_config.readiness_delay || 7 end @@ -206,7 +201,6 @@ class Kamal::Configuration raw_config.drain_timeout || 30 end - def run_directory ".kamal" end @@ -227,7 +221,6 @@ class Kamal::Configuration File.join app_directory, "assets" end - def hooks_path raw_config.hooks_path || ".kamal/hooks" end @@ -236,7 +229,6 @@ class Kamal::Configuration raw_config.asset_path end - def env_tags @env_tags ||= if (tags = raw_config.env["tags"]) tags.collect { |name, config| Env::Tag.new(name, config: config, secrets: secrets) } @@ -277,7 +269,6 @@ class Kamal::Configuration File.join proxy_directory, "options" end - def to_h { roles: role_names, diff --git a/lib/kamal/configuration/accessory.rb b/lib/kamal/configuration/accessory.rb index 198e6321..ccb845fd 100644 --- a/lib/kamal/configuration/accessory.rb +++ b/lib/kamal/configuration/accessory.rb @@ -5,7 +5,7 @@ class Kamal::Configuration::Accessory delegate :argumentize, :optionize, to: Kamal::Utils - attr_reader :name, :accessory_config, :env, :proxy + attr_reader :name, :env, :proxy, :registry def initialize(name, config:) @name, @config, @accessory_config = name.inquiry, config, config.raw_config["accessories"][name] @@ -16,12 +16,9 @@ class Kamal::Configuration::Accessory context: "accessories/#{name}", with: Kamal::Configuration::Validator::Accessory - @env = Kamal::Configuration::Env.new \ - config: accessory_config.fetch("env", {}), - secrets: config.secrets, - context: "accessories/#{name}/env" - - initialize_proxy if running_proxy? + @env = initialize_env + @proxy = initialize_proxy if running_proxy? + @registry = initialize_registry if accessory_config["registry"].present? end def service_name @@ -29,7 +26,7 @@ class Kamal::Configuration::Accessory end def image - accessory_config["image"] + [ registry&.server, accessory_config["image"] ].compact.join("/") end def hosts @@ -109,18 +106,32 @@ class Kamal::Configuration::Accessory end def running_proxy? - @accessory_config["proxy"].present? - end - - def initialize_proxy - @proxy = Kamal::Configuration::Proxy.new \ - config: config, - proxy_config: accessory_config["proxy"], - context: "accessories/#{name}/proxy" + accessory_config["proxy"].present? end private - attr_accessor :config + attr_reader :config, :accessory_config + + def initialize_env + Kamal::Configuration::Env.new \ + config: accessory_config.fetch("env", {}), + secrets: config.secrets, + context: "accessories/#{name}/env" + end + + def initialize_proxy + Kamal::Configuration::Proxy.new \ + config: config, + proxy_config: accessory_config["proxy"], + context: "accessories/#{name}/proxy" + end + + def initialize_registry + Kamal::Configuration::Registry.new \ + config: accessory_config, + secrets: config.secrets, + context: "accessories/#{name}/registry" + end def default_labels { "service" => service_name } diff --git a/lib/kamal/configuration/docs/accessory.yml b/lib/kamal/configuration/docs/accessory.yml index b82a476e..571d7217 100644 --- a/lib/kamal/configuration/docs/accessory.yml +++ b/lib/kamal/configuration/docs/accessory.yml @@ -23,9 +23,27 @@ accessories: # Image # - # The Docker image to use, prefix it with a registry if not using Docker Hub: + # The Docker image to use. + # Prefix it with its server when using root level registry different from Docker Hub. + # Define registry directly or via anchors when it differs from root level registry. image: mysql:8.0 + # Registry + # + # By default accessories use Docker Hub registry. + # You can specify different registry per accessory with this option. + # Don't prefix image with this registry server. + # Use anchors if you need to set the same specific registry for several accessories. + # + # ```yml + # registry: + # <<: *specific-registry + # ``` + # + # See kamal docs registry for more information: + registry: + ... + # Accessory hosts # # Specify one of `host`, `hosts`, or `roles`: diff --git a/lib/kamal/configuration/registry.rb b/lib/kamal/configuration/registry.rb index 763cf976..d3fba515 100644 --- a/lib/kamal/configuration/registry.rb +++ b/lib/kamal/configuration/registry.rb @@ -1,12 +1,10 @@ class Kamal::Configuration::Registry include Kamal::Configuration::Validation - attr_reader :registry_config, :secrets - - def initialize(config:) - @registry_config = config.raw_config.registry || {} - @secrets = config.secrets - validate! registry_config, with: Kamal::Configuration::Validator::Registry + def initialize(config:, secrets:, context: "registry") + @registry_config = config["registry"] || {} + @secrets = secrets + validate! registry_config, context: context, with: Kamal::Configuration::Validator::Registry end def server @@ -22,6 +20,8 @@ class Kamal::Configuration::Registry end private + attr_reader :registry_config, :secrets + def lookup(key) if registry_config[key].is_a?(Array) secrets[registry_config[key].first] diff --git a/test/cli/accessory_test.rb b/test/cli/accessory_test.rb index 05431fcb..cc517e59 100644 --- a/test/cli/accessory_test.rb +++ b/test/cli/accessory_test.rb @@ -14,8 +14,8 @@ class CliAccessoryTest < CliTestCase Kamal::Cli::Accessory.any_instance.expects(:upload).with("mysql") run_command("boot", "mysql").tap do |output| - assert_match /docker login.*on 1.1.1.3/, output - assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" private.registry/mysql:5.7 on 1.1.1.3", output end end @@ -24,17 +24,21 @@ class CliAccessoryTest < CliTestCase Kamal::Cli::Accessory.any_instance.expects(:upload).with("mysql") Kamal::Cli::Accessory.any_instance.expects(:directories).with("redis") Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis") + Kamal::Cli::Accessory.any_instance.expects(:directories).with("busybox") + Kamal::Cli::Accessory.any_instance.expects(:upload).with("busybox") run_command("boot", "all").tap do |output| - assert_match /docker login.*on 1.1.1.3/, output - assert_match /docker login.*on 1.1.1.1/, output - assert_match /docker login.*on 1.1.1.2/, output + assert_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.3", output + assert_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.1", output + assert_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.2", output + assert_match "docker login other.registry -u [REDACTED] -p [REDACTED] on 1.1.1.3", output assert_match /docker network create kamal.*on 1.1.1.1/, output assert_match /docker network create kamal.*on 1.1.1.2/, output assert_match /docker network create kamal.*on 1.1.1.3/, output - assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" private.registry/mysql:5.7 on 1.1.1.3", output assert_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output assert_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output + assert_match "docker run --name custom-box --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --env-file .kamal/apps/app/env/accessories/busybox.env --label service=\"custom-box\" other.registry/busybox:latest on 1.1.1.3", output end end @@ -60,13 +64,16 @@ class CliAccessoryTest < CliTestCase end test "reboot all" do - Kamal::Commands::Registry.any_instance.expects(:login).times(3) + Kamal::Commands::Registry.any_instance.expects(:login).times(4) Kamal::Cli::Accessory.any_instance.expects(:stop).with("mysql") Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("mysql") Kamal::Cli::Accessory.any_instance.expects(:boot).with("mysql", prepare: false) Kamal::Cli::Accessory.any_instance.expects(:stop).with("redis") Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("redis") Kamal::Cli::Accessory.any_instance.expects(:boot).with("redis", prepare: false) + Kamal::Cli::Accessory.any_instance.expects(:stop).with("busybox") + Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("busybox") + Kamal::Cli::Accessory.any_instance.expects(:boot).with("busybox", prepare: false) run_command("reboot", "all") end @@ -94,7 +101,7 @@ class CliAccessoryTest < CliTestCase end test "details with non-existent accessory" do - assert_equal "No accessory by the name of 'hello' (options: mysql and redis)", stderred { run_command("details", "hello") } + assert_equal "No accessory by the name of 'hello' (options: mysql, redis, and busybox)", stderred { run_command("details", "hello") } end test "details with all" do @@ -180,6 +187,10 @@ class CliAccessoryTest < CliTestCase Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("redis") Kamal::Cli::Accessory.any_instance.expects(:remove_image).with("redis") Kamal::Cli::Accessory.any_instance.expects(:remove_service_directory).with("redis") + Kamal::Cli::Accessory.any_instance.expects(:stop).with("busybox") + Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("busybox") + Kamal::Cli::Accessory.any_instance.expects(:remove_image).with("busybox") + Kamal::Cli::Accessory.any_instance.expects(:remove_service_directory).with("busybox") run_command("remove", "all", "-y") end @@ -189,7 +200,7 @@ class CliAccessoryTest < CliTestCase end test "remove_image" do - assert_match "docker image rm --force mysql", run_command("remove_image", "mysql") + assert_match "docker image rm --force private.registry/mysql:5.7", run_command("remove_image", "mysql") end test "remove_service_directory" do @@ -201,8 +212,8 @@ class CliAccessoryTest < CliTestCase Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis") run_command("boot", "redis", "--hosts", "1.1.1.1").tap do |output| - assert_match /docker login.*on 1.1.1.1/, output - assert_no_match /docker login.*on 1.1.1.2/, output + assert_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.1", output + assert_no_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.2", output assert_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output assert_no_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output end @@ -213,8 +224,8 @@ class CliAccessoryTest < CliTestCase Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis") run_command("boot", "redis", "--hosts", "1.1.1.1,1.1.1.3").tap do |output| - assert_match /docker login.*on 1.1.1.1/, output - assert_no_match /docker login.*on 1.1.1.3/, output + assert_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.1", output + assert_no_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.3", output assert_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output assert_no_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.3", output end @@ -225,7 +236,7 @@ class CliAccessoryTest < CliTestCase assert_match "Upgrading all accessories on 1.1.1.3,1.1.1.1,1.1.1.2...", output assert_match "docker network create kamal on 1.1.1.3", output assert_match "docker container stop app-mysql on 1.1.1.3", output - assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST="%" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST="%" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" private.registry/mysql:5.7 on 1.1.1.3", output assert_match "Upgraded all accessories on 1.1.1.3,1.1.1.1,1.1.1.2...", output end end @@ -235,14 +246,13 @@ class CliAccessoryTest < CliTestCase assert_match "Upgrading all accessories on 1.1.1.3...", output assert_match "docker network create kamal on 1.1.1.3", output assert_match "docker container stop app-mysql on 1.1.1.3", output - assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST="%" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST="%" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" private.registry/mysql:5.7 on 1.1.1.3", output assert_match "Upgraded all accessories on 1.1.1.3", output end end - private def run_command(*command) - stdouted { Kamal::Cli::Accessory.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) } + stdouted { Kamal::Cli::Accessory.start([ *command, "-c", "test/fixtures/deploy_with_accessories_with_different_registries.yml" ]) } end end diff --git a/test/commands/accessory_test.rb b/test/commands/accessory_test.rb index b9bcca7e..6ff9902e 100644 --- a/test/commands/accessory_test.rb +++ b/test/commands/accessory_test.rb @@ -5,7 +5,9 @@ class CommandsAccessoryTest < ActiveSupport::TestCase setup_test_secrets("secrets" => "MYSQL_ROOT_PASSWORD=secret123") @config = { - service: "app", image: "dhh/app", registry: { "server" => "private.registry", "username" => "dhh", "password" => "secret" }, + service: "app", + image: "dhh/app", + registry: { "server" => "private.registry", "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], builder: { "arch" => "amd64" }, accessories: { @@ -39,6 +41,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase "busybox" => { "service" => "custom-busybox", "image" => "busybox:latest", + "registry" => { "server" => "other.registry", "username" => "user", "password" => "pw" }, "host" => "1.1.1.7", "proxy" => { "host" => "busybox.example.com" @@ -62,7 +65,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase new_command(:redis).run.join(" ") assert_equal \ - "docker run --name custom-busybox --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --env-file .kamal/apps/app/env/accessories/busybox.env --label service=\"custom-busybox\" busybox:latest", + "docker run --name custom-busybox --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --env-file .kamal/apps/app/env/accessories/busybox.env --label service=\"custom-busybox\" other.registry/busybox:latest", new_command(:busybox).run.join(" ") end @@ -70,7 +73,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } assert_equal \ - "docker run --name custom-busybox --detach --restart unless-stopped --network kamal --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --env-file .kamal/apps/app/env/accessories/busybox.env --label service=\"custom-busybox\" busybox:latest", + "docker run --name custom-busybox --detach --restart unless-stopped --network kamal --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --env-file .kamal/apps/app/env/accessories/busybox.env --label service=\"custom-busybox\" other.registry/busybox:latest", new_command(:busybox).run.join(" ") end @@ -100,7 +103,6 @@ class CommandsAccessoryTest < ActiveSupport::TestCase new_command(:mysql).info.join(" ") end - test "execute in new container" do assert_equal \ "docker run --rm --network kamal --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/apps/app/env/accessories/mysql.env private.registry/mysql:8.0 mysql -u root", @@ -127,8 +129,6 @@ class CommandsAccessoryTest < ActiveSupport::TestCase end end - - test "logs" do assert_equal \ "docker logs app-mysql --timestamps 2>&1", diff --git a/test/commands/registry_test.rb b/test/commands/registry_test.rb index cf2734b7..0a71b1da 100755 --- a/test/commands/registry_test.rb +++ b/test/commands/registry_test.rb @@ -2,14 +2,27 @@ require "test_helper" class CommandsRegistryTest < ActiveSupport::TestCase setup do - @config = { service: "app", + @config = { + service: "app", image: "dhh/app", - registry: { "username" => "dhh", + registry: { + "username" => "dhh", "password" => "secret", "server" => "hub.docker.com" }, builder: { "arch" => "amd64" }, - servers: [ "1.1.1.1" ] + servers: [ "1.1.1.1" ], + accessories: { + "db" => { + "image" => "mysql:8.0", + "hosts" => [ "1.1.1.1" ], + "registry" => { + "username" => "user", + "password" => "pw", + "server" => "other.hub.docker.com" + } + } + } } end @@ -19,13 +32,24 @@ class CommandsRegistryTest < ActiveSupport::TestCase registry.login.join(" ") end + test "given registry login" do + assert_equal \ + "docker login other.hub.docker.com -u \"user\" -p \"pw\"", + registry.login(registry_config: accessory_registry_config).join(" ") + end + test "registry login with ENV password" do - with_test_secrets("secrets" => "KAMAL_REGISTRY_PASSWORD=more-secret") do + with_test_secrets("secrets" => "KAMAL_REGISTRY_PASSWORD=more-secret\nKAMAL_MYSQL_REGISTRY_PASSWORD=secret-pw") do @config[:registry]["password"] = [ "KAMAL_REGISTRY_PASSWORD" ] + @config[:accessories]["db"]["registry"]["password"] = [ "KAMAL_MYSQL_REGISTRY_PASSWORD" ] assert_equal \ "docker login hub.docker.com -u \"dhh\" -p \"more-secret\"", registry.login.join(" ") + + assert_equal \ + "docker login other.hub.docker.com -u \"user\" -p \"secret-pw\"", + registry.login(registry_config: accessory_registry_config).join(" ") end end @@ -55,8 +79,22 @@ class CommandsRegistryTest < ActiveSupport::TestCase registry.logout.join(" ") end + test "given registry logout" do + assert_equal \ + "docker logout other.hub.docker.com", + registry.logout(registry_config: accessory_registry_config).join(" ") + end + private def registry - Kamal::Commands::Registry.new Kamal::Configuration.new(@config) + Kamal::Commands::Registry.new main_config + end + + def main_config + Kamal::Configuration.new(@config) + end + + def accessory_registry_config + main_config.accessory("db").registry end end diff --git a/test/configuration/accessory_test.rb b/test/configuration/accessory_test.rb index d15a48ad..fc01dc90 100644 --- a/test/configuration/accessory_test.rb +++ b/test/configuration/accessory_test.rb @@ -3,7 +3,9 @@ require "test_helper" class ConfigurationAccessoryTest < ActiveSupport::TestCase setup do @deploy = { - service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, + service: "app", + image: "dhh/app", + registry: { "username" => "dhh", "password" => "secret" }, servers: { "web" => [ "1.1.1.1", "1.1.1.2" ], "workers" => [ "1.1.1.3", "1.1.1.4" ] @@ -12,7 +14,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase env: { "REDIS_URL" => "redis://x/y" }, accessories: { "mysql" => { - "image" => "mysql:8.0", + "image" => "public.registry/mysql:8.0", "host" => "1.1.1.5", "port" => "3306", "env" => { @@ -52,6 +54,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase "monitoring" => { "service" => "custom-monitoring", "image" => "monitoring:latest", + "registry" => { "server" => "other.registry", "username" => "user", "password" => "pw" }, "roles" => [ "web" ], "port" => "4321:4321", "labels" => { @@ -80,6 +83,21 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase assert_equal "custom-monitoring", @config.accessory(:monitoring).service_name end + test "image" do + assert_equal "public.registry/mysql:8.0", @config.accessory(:mysql).image + assert_equal "redis:latest", @config.accessory(:redis).image + assert_equal "other.registry/monitoring:latest", @config.accessory(:monitoring).image + end + + test "registry" do + assert_nil @config.accessory(:mysql).registry + assert_nil @config.accessory(:redis).registry + monitoring_registry = @config.accessory(:monitoring).registry + assert_equal "other.registry", monitoring_registry.server + assert_equal "user", monitoring_registry.username + assert_equal "pw", monitoring_registry.password + end + test "port" do assert_equal "3306:3306", @config.accessory(:mysql).port assert_equal "6379:6379", @config.accessory(:redis).port diff --git a/test/fixtures/deploy_with_accessories_with_different_registries.yml b/test/fixtures/deploy_with_accessories_with_different_registries.yml new file mode 100644 index 00000000..8bd8f1d9 --- /dev/null +++ b/test/fixtures/deploy_with_accessories_with_different_registries.yml @@ -0,0 +1,47 @@ +service: app +image: dhh/app +servers: + web: + - "1.1.1.1" + - "1.1.1.2" + workers: + - "1.1.1.3" + - "1.1.1.4" +registry: + server: private.registry + username: user + password: pw +builder: + arch: amd64 + +accessories: + mysql: + image: private.registry/mysql:5.7 + host: 1.1.1.3 + port: 3306 + env: + clear: + MYSQL_ROOT_HOST: '%' + secret: + - MYSQL_ROOT_PASSWORD + files: + - test/fixtures/files/my.cnf:/etc/mysql/my.cnf + directories: + - data:/var/lib/mysql + redis: + image: redis:latest + roles: + - web + port: 6379 + directories: + - data:/data + busybox: + service: custom-box + image: busybox:latest + host: 1.1.1.3 + registry: + server: other.registry + username: other_user + password: other_pw + +readiness_delay: 0 From bd8689c185d9ed9d25fbc52ec18b167eb66a869a Mon Sep 17 00:00:00 2001 From: Pavel Shpak Date: Sun, 22 Dec 2024 03:28:12 +0200 Subject: [PATCH 28/43] Fix bug in role validate_servers. There were typo-bug during `validate_servers!` invocation for role. It wasn't discovered, because it never met condition. Because role_config wasn't correctly extracted for validation. Also remove not used anymore `accessories_on`. Leftover from previous changes. --- lib/kamal/commander.rb | 6 ------ lib/kamal/configuration/role.rb | 12 ++++++------ lib/kamal/configuration/validator/role.rb | 2 +- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/lib/kamal/commander.rb b/lib/kamal/commander.rb index 1557df57..6a461276 100644 --- a/lib/kamal/commander.rb +++ b/lib/kamal/commander.rb @@ -76,11 +76,6 @@ class Kamal::Commander config.accessories&.collect(&:name) || [] end - def accessories_on(host) - config.accessories.select { |accessory| accessory.hosts.include?(host.to_s) }.map(&:name) - end - - def app(role: nil, host: nil) Kamal::Commands::App.new(config, role: role, host: host) end @@ -129,7 +124,6 @@ class Kamal::Commander config.aliases[name] end - def with_verbosity(level) old_level = self.verbosity diff --git a/lib/kamal/configuration/role.rb b/lib/kamal/configuration/role.rb index 708e77fc..c6bd8783 100644 --- a/lib/kamal/configuration/role.rb +++ b/lib/kamal/configuration/role.rb @@ -10,7 +10,7 @@ class Kamal::Configuration::Role def initialize(name, config:) @name, @config = name.inquiry, config validate! \ - specializations, + role_config, example: validation_yml["servers"]["workers"], context: "servers/#{name}", with: Kamal::Configuration::Validator::Role @@ -204,11 +204,11 @@ class Kamal::Configuration::Role end def specializations - if config.raw_config.servers.is_a?(Array) || config.raw_config.servers[name].is_a?(Array) - {} - else - config.raw_config.servers[name] - end + @specializations ||= role_config.is_a?(Array) ? {} : role_config + end + + def role_config + @role_config ||= config.raw_config.servers.is_a?(Array) ? {} : config.raw_config.servers[name] end def custom_labels diff --git a/lib/kamal/configuration/validator/role.rb b/lib/kamal/configuration/validator/role.rb index ce28c039..de7a1969 100644 --- a/lib/kamal/configuration/validator/role.rb +++ b/lib/kamal/configuration/validator/role.rb @@ -3,7 +3,7 @@ class Kamal::Configuration::Validator::Role < Kamal::Configuration::Validator validate_type! config, Array, Hash if config.is_a?(Array) - validate_servers! "servers", config + validate_servers!(config) else super end From d59c274208c3b52655c69c7402be34e2d6856f56 Mon Sep 17 00:00:00 2001 From: Pavel Shpak Date: Sun, 22 Dec 2024 04:37:15 +0200 Subject: [PATCH 29/43] Fix typo in configuration initializer method. --- lib/kamal/configuration.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index 827e1204..90c3f8cd 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -59,7 +59,7 @@ class Kamal::Configuration # Eager load config to validate it, these are first as they have dependencies later on @servers = Servers.new(config: self) - @registry = Registry.new(config: raw_config, secrets: secrets) + @registry = Registry.new(config: @raw_config, secrets: secrets) @accessories = @raw_config.accessories&.keys&.collect { |name| Accessory.new(name, config: self) } || [] @aliases = @raw_config.aliases&.keys&.to_h { |name| [ name, Alias.new(name, config: self) ] } || {} From 78fcc3d88f168a0ca52ee3876e13252468144dc4 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Wed, 2 Oct 2024 16:09:26 +0100 Subject: [PATCH 30/43] Allow destination and config-file in aliases We only loaded the configuration once, which meant that aliases always used the initial configuration file and destination. We don't want to load the configuration in subcommands as it is not passed all the options we need. But just checking if we are in a subcommand is enough - the alias reloads and the subcommand does not. One thing to note is that anything passed on the command line overrides what is in the alias, so if an alias says `other_config: config -c config/deploy2.yml` and you run `kamal other_config -c config/deploy.yml`, it won't switch. --- lib/kamal/cli/base.rb | 3 +- test/cli/app_test.rb | 6 ++-- test/cli/build_test.rb | 13 --------- test/cli/cli_test_case.rb | 13 +++++++++ test/cli/main_test.rb | 42 +++++++++++++++++++++++++++ test/fixtures/deploy.elsewhere.yml | 12 ++++++++ test/fixtures/deploy.yml | 13 +++++++++ test/fixtures/deploy2.yml | 12 ++++++++ test/fixtures/deploy_with_aliases.yml | 3 ++ 9 files changed, 101 insertions(+), 16 deletions(-) create mode 100644 test/fixtures/deploy.elsewhere.yml create mode 100644 test/fixtures/deploy.yml create mode 100644 test/fixtures/deploy2.yml diff --git a/lib/kamal/cli/base.rb b/lib/kamal/cli/base.rb index 4aebfd90..9f304cdd 100644 --- a/lib/kamal/cli/base.rb +++ b/lib/kamal/cli/base.rb @@ -30,7 +30,8 @@ module Kamal::Cli else super end - initialize_commander unless KAMAL.configured? + + initialize_commander unless config[:invoked_via_subcommand] end private diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 2e532730..408579de 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -382,8 +382,10 @@ class CliAppTest < CliTestCase test "version through main" do - stdouted { Kamal::Cli::Main.start([ "app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1" ]) }.tap do |output| - assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output + with_argv([ "app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1" ]) do + stdouted { Kamal::Cli::Main.start }.tap do |output| + assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output + end end end diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb index 4259fa5b..88f00743 100644 --- a/test/cli/build_test.rb +++ b/test/cli/build_test.rb @@ -274,17 +274,4 @@ class CliBuildTest < CliTestCase SSHKit::Backend::Abstract.any_instance.stubs(:execute) .with { |*args| args[0..1] == [ :docker, :buildx ] } end - - def with_build_directory - build_directory = File.join Dir.tmpdir, "kamal-clones", "app-#{pwd_sha}", "kamal" - FileUtils.mkdir_p build_directory - FileUtils.touch File.join build_directory, "Dockerfile" - yield build_directory + "/" - ensure - FileUtils.rm_rf build_directory - end - - def pwd_sha - Digest::SHA256.hexdigest(Dir.pwd)[0..12] - end end diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb index 27bf7b69..d4b57923 100644 --- a/test/cli/cli_test_case.rb +++ b/test/cli/cli_test_case.rb @@ -51,4 +51,17 @@ class CliTestCase < ActiveSupport::TestCase ensure ARGV.replace(old_argv) end + + def with_build_directory + build_directory = File.join Dir.tmpdir, "kamal-clones", "app-#{pwd_sha}", "kamal" + FileUtils.mkdir_p build_directory + FileUtils.touch File.join build_directory, "Dockerfile" + yield build_directory + "/" + ensure + FileUtils.rm_rf build_directory + end + + def pwd_sha + Digest::SHA256.hexdigest(Dir.pwd)[0..12] + end end diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index cd0efe1f..8ecb1cfa 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -460,6 +460,7 @@ class CliMainTest < CliTestCase test "run an alias for a console" do run_command("console", config_file: "deploy_with_aliases").tap do |output| + assert_no_match "App Host: 1.1.1.4", output assert_match "docker exec app-console-999 bin/console on 1.1.1.5", output assert_match "App Host: 1.1.1.5", output end @@ -486,6 +487,33 @@ class CliMainTest < CliTestCase end end + test "switch config file with an alias" do + with_config_files do + with_argv([ "other_config" ]) do + stdouted { Kamal::Cli::Main.start }.tap do |output| + assert_match ":service_with_version: app2-999", output + end + end + end + end + + test "switch destination with an alias" do + with_config_files do + with_argv([ "other_destination_config" ]) do + stdouted { Kamal::Cli::Main.start }.tap do |output| + assert_match ":service_with_version: app3-999", output + end + end + end + end + + test "run on primary via alias" do + run_command("primary_details", config_file: "deploy_with_aliases").tap do |output| + assert_match "App Host: 1.1.1.1", output + assert_no_match "App Host: 1.1.1.2", output + end + end + test "upgrade" do invoke_options = { "config_file" => "test/fixtures/deploy_with_accessories.yml", "skip_hooks" => false, "confirmed" => true, "rolling" => false } Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:upgrade", [], invoke_options) @@ -530,6 +558,20 @@ class CliMainTest < CliTestCase end end + def with_config_files + Dir.mktmpdir do |tmpdir| + config_dir = File.join(tmpdir, "config") + FileUtils.mkdir_p(config_dir) + FileUtils.cp "test/fixtures/deploy.yml", config_dir + FileUtils.cp "test/fixtures/deploy2.yml", config_dir + FileUtils.cp "test/fixtures/deploy.elsewhere.yml", config_dir + + Dir.chdir(tmpdir) do + yield + end + end + end + def assert_file(file, content) assert_match content, File.read(file) end diff --git a/test/fixtures/deploy.elsewhere.yml b/test/fixtures/deploy.elsewhere.yml new file mode 100644 index 00000000..479e0f9d --- /dev/null +++ b/test/fixtures/deploy.elsewhere.yml @@ -0,0 +1,12 @@ +service: app3 +image: dhh/app3 +servers: + - "1.1.1.3" + - "1.1.1.4" +registry: + username: user + password: pw +builder: + arch: amd64 +aliases: + other_config: config -c config/deploy2.yml diff --git a/test/fixtures/deploy.yml b/test/fixtures/deploy.yml new file mode 100644 index 00000000..c532000f --- /dev/null +++ b/test/fixtures/deploy.yml @@ -0,0 +1,13 @@ +service: app +image: dhh/app +servers: + - "1.1.1.1" + - "1.1.1.2" +registry: + username: user + password: pw +builder: + arch: amd64 +aliases: + other_config: config -c config/deploy2.yml + other_destination_config: config -d elsewhere diff --git a/test/fixtures/deploy2.yml b/test/fixtures/deploy2.yml new file mode 100644 index 00000000..14b56951 --- /dev/null +++ b/test/fixtures/deploy2.yml @@ -0,0 +1,12 @@ +service: app2 +image: dhh/app2 +servers: + - "1.1.1.1" + - "1.1.1.2" +registry: + username: user2 + password: pw2 +builder: + arch: amd64 +aliases: + other_config: config -c config/deploy2.yml diff --git a/test/fixtures/deploy_with_aliases.yml b/test/fixtures/deploy_with_aliases.yml index ec7b14a0..104fc462 100644 --- a/test/fixtures/deploy_with_aliases.yml +++ b/test/fixtures/deploy_with_aliases.yml @@ -21,3 +21,6 @@ aliases: console: app exec --reuse -p -r console "bin/console" exec: app exec --reuse -p -r console rails: app exec --reuse -p -r console rails + primary_details: details -p + deploy_secondary: deploy -d secondary + From 2cdca4596c4d0fbe352a51524ab894d9c43a523e Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Thu, 16 Jan 2025 16:28:02 +0000 Subject: [PATCH 31/43] Create but don't run the assets container We don't need to run the assets container to copy the assets out, instead we can just create, copy and remove. --- lib/kamal/commands/app/assets.rb | 8 ++++---- test/cli/app_test.rb | 2 +- test/commands/app_test.rb | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/kamal/commands/app/assets.rb b/lib/kamal/commands/app/assets.rb index c1e65d18..21ae4d5f 100644 --- a/lib/kamal/commands/app/assets.rb +++ b/lib/kamal/commands/app/assets.rb @@ -4,10 +4,10 @@ module Kamal::Commands::App::Assets combine \ make_directory(role.asset_extracted_directory), - [ *docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true" ], - docker(:run, "--name", asset_container, "--detach", "--rm", "--entrypoint", "sleep", config.absolute_image, "1000000"), - docker(:cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_directory), - docker(:stop, "-t 1", asset_container), + [ *docker(:container, :rm, asset_container, "2> /dev/null"), "|| true" ], + docker(:container, :create, "--name", asset_container, config.absolute_image), + docker(:container, :cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_directory), + docker(:container, :rm, asset_container), by: "&&" end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 2e532730..e6505a45 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -73,7 +73,7 @@ class CliAppTest < CliTestCase run_command("boot", config: :with_assets).tap do |output| assert_match "docker tag dhh/app:latest dhh/app:latest", output assert_match "/usr/bin/env mkdir -p .kamal/apps/app/assets/volumes/web-latest ; cp -rnT .kamal/apps/app/assets/extracted/web-latest .kamal/apps/app/assets/volumes/web-latest ; cp -rnT .kamal/apps/app/assets/extracted/web-latest .kamal/apps/app/assets/volumes/web-123 || true ; cp -rnT .kamal/apps/app/assets/extracted/web-123 .kamal/apps/app/assets/volumes/web-latest || true", output - assert_match "/usr/bin/env mkdir -p .kamal/apps/app/assets/extracted/web-latest && docker stop -t 1 app-web-assets 2> /dev/null || true && docker run --name app-web-assets --detach --rm --entrypoint sleep dhh/app:latest 1000000 && docker cp -L app-web-assets:/public/assets/. .kamal/apps/app/assets/extracted/web-latest && docker stop -t 1 app-web-assets", output + assert_match "/usr/bin/env mkdir -p .kamal/apps/app/assets/extracted/web-latest && docker container rm app-web-assets 2> /dev/null || true && docker container create --name app-web-assets dhh/app:latest && docker container cp -L app-web-assets:/public/assets/. .kamal/apps/app/assets/extracted/web-latest && docker container rm app-web-assets", output assert_match /docker run --detach --restart unless-stopped --name app-web-latest --network kamal --hostname 1.1.1.1-[0-9a-f]{12} /, output assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output assert_match "/usr/bin/env find .kamal/apps/app/assets/extracted -maxdepth 1 -name 'web-*' ! -name web-latest -exec rm -rf \"{}\" + ; find .kamal/apps/app/assets/volumes -maxdepth 1 -name 'web-*' ! -name web-latest -exec rm -rf \"{}\" +", output diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index d53b31e1..75241597 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -469,10 +469,10 @@ class CommandsAppTest < ActiveSupport::TestCase test "extract assets" do assert_equal [ :mkdir, "-p", ".kamal/apps/app/assets/extracted/web-999", "&&", - :docker, :stop, "-t 1", "app-web-assets", "2> /dev/null", "|| true", "&&", - :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "--entrypoint", "sleep", "dhh/app:999", "1000000", "&&", - :docker, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/apps/app/assets/extracted/web-999", "&&", - :docker, :stop, "-t 1", "app-web-assets" + :docker, :container, :rm, "app-web-assets", "2> /dev/null", "|| true", "&&", + :docker, :container, :create, "--name", "app-web-assets", "dhh/app:999", "&&", + :docker, :container, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/apps/app/assets/extracted/web-999", "&&", + :docker, :container, :rm, "app-web-assets" ], new_command(asset_path: "/public/assets").extract_assets end From dd8cadf7437db0a4c41479326dc5c65a1f06ebe2 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 17 Jan 2025 11:06:29 +0000 Subject: [PATCH 32/43] Add tests for env/secret file precedence --- test/secrets_test.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/secrets_test.rb b/test/secrets_test.rb index aca9cebe..f0ca7e29 100644 --- a/test/secrets_test.rb +++ b/test/secrets_test.rb @@ -20,6 +20,20 @@ class SecretsTest < ActiveSupport::TestCase end end + test "env references" do + with_test_secrets("secrets" => "SECRET1=$SECRET1") do + ENV["SECRET1"] = "ABC" + assert_equal "ABC", Kamal::Secrets.new["SECRET1"] + end + end + + test "secrets file value overrides env" do + with_test_secrets("secrets" => "SECRET1=DEF") do + ENV["SECRET1"] = "ABC" + assert_equal "DEF", Kamal::Secrets.new["SECRET1"] + end + end + test "destinations" do with_test_secrets("secrets.dest" => "SECRET=DEF", "secrets" => "SECRET=ABC", "secrets-common" => "SECRET=GHI\nSECRET2=JKL") do assert_equal "ABC", Kamal::Secrets.new["SECRET"] From db7556ed9953f79dfee7d831d107127ca6361b9b Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 17 Jan 2025 12:05:28 +0000 Subject: [PATCH 33/43] Fix enpass adapter There were changes in main that meant the tests failed after merging. Adding the new `requires_account?` method to the enpass adapter fixed it. --- lib/kamal/secrets/adapters/enpass.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/kamal/secrets/adapters/enpass.rb b/lib/kamal/secrets/adapters/enpass.rb index 68fe41a9..17788f1d 100644 --- a/lib/kamal/secrets/adapters/enpass.rb +++ b/lib/kamal/secrets/adapters/enpass.rb @@ -14,6 +14,10 @@ class Kamal::Secrets::Adapters::Enpass < Kamal::Secrets::Adapters::Base fetch_secrets(secrets, from) end + def requires_account? + false + end + private def fetch_secrets(secrets, vault) secrets_titles = fetch_secret_titles(secrets) From a1708f687f6f3a5bb16b4f65e57fb3d311cc0c0b Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 17 Jan 2025 12:24:46 +0000 Subject: [PATCH 34/43] Prefix secrets in fetch_secrets This allows us to remove the custom fetch method for enpass. --- .../secrets/adapters/aws_secrets_manager.rb | 4 ++-- lib/kamal/secrets/adapters/base.rb | 7 ++++-- lib/kamal/secrets/adapters/bitwarden.rb | 4 ++-- .../adapters/bitwarden_secrets_manager.rb | 23 +++++++++++-------- lib/kamal/secrets/adapters/doppler.rb | 4 +++- lib/kamal/secrets/adapters/enpass.rb | 13 +++++------ .../secrets/adapters/gcp_secret_manager.rb | 4 ++-- lib/kamal/secrets/adapters/last_pass.rb | 3 ++- lib/kamal/secrets/adapters/one_password.rb | 4 ++-- lib/kamal/secrets/adapters/test.rb | 4 ++-- 10 files changed, 40 insertions(+), 30 deletions(-) diff --git a/lib/kamal/secrets/adapters/aws_secrets_manager.rb b/lib/kamal/secrets/adapters/aws_secrets_manager.rb index 4bcac21d..48add1ac 100644 --- a/lib/kamal/secrets/adapters/aws_secrets_manager.rb +++ b/lib/kamal/secrets/adapters/aws_secrets_manager.rb @@ -4,9 +4,9 @@ class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Ba nil end - def fetch_secrets(secrets, account:, session:) + def fetch_secrets(secrets, from:, account:, session:) {}.tap do |results| - get_from_secrets_manager(secrets, account: account).each do |secret| + get_from_secrets_manager(prefixed_secrets(secrets, from: from), account: account).each do |secret| secret_name = secret["Name"] secret_string = JSON.parse(secret["SecretString"]) diff --git a/lib/kamal/secrets/adapters/base.rb b/lib/kamal/secrets/adapters/base.rb index fc66bb34..c74f7c41 100644 --- a/lib/kamal/secrets/adapters/base.rb +++ b/lib/kamal/secrets/adapters/base.rb @@ -7,8 +7,7 @@ class Kamal::Secrets::Adapters::Base check_dependencies! session = login(account) - full_secrets = secrets.map { |secret| [ from, secret ].compact.join("/") } - fetch_secrets(full_secrets, account: account, session: session) + fetch_secrets(secrets, from: from, account: account, session: session) end def requires_account? @@ -27,4 +26,8 @@ class Kamal::Secrets::Adapters::Base def check_dependencies! raise NotImplementedError end + + def prefixed_secrets(secrets, from:) + secrets.map { |secret| [ from, secret ].compact.join("/") } + end end diff --git a/lib/kamal/secrets/adapters/bitwarden.rb b/lib/kamal/secrets/adapters/bitwarden.rb index 5dfb72db..6bd4fb25 100644 --- a/lib/kamal/secrets/adapters/bitwarden.rb +++ b/lib/kamal/secrets/adapters/bitwarden.rb @@ -21,9 +21,9 @@ class Kamal::Secrets::Adapters::Bitwarden < Kamal::Secrets::Adapters::Base session end - def fetch_secrets(secrets, account:, session:) + def fetch_secrets(secrets, from:, account:, session:) {}.tap do |results| - items_fields(secrets).each do |item, fields| + items_fields(prefixed_secrets(secrets, from: from)).each do |item, fields| item_json = run_command("get item #{item.shellescape}", session: session, raw: true) raise RuntimeError, "Could not read #{item} from Bitwarden" unless $?.success? item_json = JSON.parse(item_json) diff --git a/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb b/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb index f0a19caa..7cbc093e 100644 --- a/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb +++ b/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb @@ -9,17 +9,11 @@ class Kamal::Secrets::Adapters::BitwardenSecretsManager < Kamal::Secrets::Adapte LIST_COMMAND = "secret list -o env" GET_COMMAND = "secret get -o env" - def fetch_secrets(secrets, account:, session:) + def fetch_secrets(secrets, from:, account:, session:) raise RuntimeError, "You must specify what to retrieve from Bitwarden Secrets Manager" if secrets.length == 0 - if secrets.length == 1 - if secrets[0] == LIST_ALL_SELECTOR - command = LIST_COMMAND - elsif secrets[0].end_with?(LIST_ALL_FROM_PROJECT_SUFFIX) - project = secrets[0].split(LIST_ALL_FROM_PROJECT_SUFFIX).first - command = "#{LIST_COMMAND} #{project}" - end - end + secrets = prefixed_secrets(secrets, from: from) + command, project = extract_command_and_project(secrets) {}.tap do |results| if command.nil? @@ -40,6 +34,17 @@ class Kamal::Secrets::Adapters::BitwardenSecretsManager < Kamal::Secrets::Adapte end end + def extract_command_and_project(secrets) + if secrets.length == 1 + if secrets[0] == LIST_ALL_SELECTOR + [ LIST_COMMAND, nil ] + elsif secrets[0].end_with?(LIST_ALL_FROM_PROJECT_SUFFIX) + project = secrets[0].split(LIST_ALL_FROM_PROJECT_SUFFIX).first + [ "#{LIST_COMMAND} #{project}", project ] + end + end + end + def parse_secret(secret) key, value = secret.split("=", 2) value = value.gsub(/^"|"$/, "") diff --git a/lib/kamal/secrets/adapters/doppler.rb b/lib/kamal/secrets/adapters/doppler.rb index 64d644f7..40eeeab5 100644 --- a/lib/kamal/secrets/adapters/doppler.rb +++ b/lib/kamal/secrets/adapters/doppler.rb @@ -16,8 +16,10 @@ class Kamal::Secrets::Adapters::Doppler < Kamal::Secrets::Adapters::Base $?.success? end - def fetch_secrets(secrets, **) + def fetch_secrets(secrets, from:, **) + secrets = prefixed_secrets(secrets, from: from) project_and_config_flags = "" + unless service_token_set? project, config, _ = secrets.first.split("/") diff --git a/lib/kamal/secrets/adapters/enpass.rb b/lib/kamal/secrets/adapters/enpass.rb index 17788f1d..96dea11a 100644 --- a/lib/kamal/secrets/adapters/enpass.rb +++ b/lib/kamal/secrets/adapters/enpass.rb @@ -9,20 +9,15 @@ # Fetch only DB_PASSWORD from FooBar item # `kamal secrets fetch --adapter enpass --from /Users/YOUR_USERNAME/Library/Containers/in.sinew.Enpass-Desktop/Data/Documents/Vaults/primary FooBar/DB_PASSWORD` class Kamal::Secrets::Adapters::Enpass < Kamal::Secrets::Adapters::Base - def fetch(secrets, account: nil, from:) - check_dependencies! - fetch_secrets(secrets, from) - end - def requires_account? false end private - def fetch_secrets(secrets, vault) + def fetch_secrets(secrets, from:, account:, session:) secrets_titles = fetch_secret_titles(secrets) - result = `enpass-cli -json -vault #{vault.shellescape} show #{secrets_titles.map(&:shellescape).join(" ")}`.strip + result = `enpass-cli -json -vault #{from.shellescape} show #{secrets_titles.map(&:shellescape).join(" ")}`.strip parse_result_and_take_secrets(result, secrets) end @@ -36,6 +31,10 @@ class Kamal::Secrets::Adapters::Enpass < Kamal::Secrets::Adapters::Base $?.success? end + def login(account) + nil + end + def fetch_secret_titles(secrets) secrets.reduce(Set.new) do |secret_titles, secret| # Sometimes secrets contain a '/', when the intent is to fetch a single password for an item. Example: FooBar/DB_PASSWORD diff --git a/lib/kamal/secrets/adapters/gcp_secret_manager.rb b/lib/kamal/secrets/adapters/gcp_secret_manager.rb index b8dfebf3..9f4ed41c 100644 --- a/lib/kamal/secrets/adapters/gcp_secret_manager.rb +++ b/lib/kamal/secrets/adapters/gcp_secret_manager.rb @@ -26,11 +26,11 @@ class Kamal::Secrets::Adapters::GcpSecretManager < Kamal::Secrets::Adapters::Bas nil end - def fetch_secrets(secrets, account:, session:) + def fetch_secrets(secrets, from:, account:, session:) user, service_account = parse_account(account) {}.tap do |results| - secrets_with_metadata(secrets).each do |secret, (project, secret_name, secret_version)| + secrets_with_metadata(prefixed_secrets(secrets, from: from)).each do |secret, (project, secret_name, secret_version)| item_name = "#{project}/#{secret_name}" results[item_name] = fetch_secret(project, secret_name, secret_version, user, service_account) raise RuntimeError, "Could not read #{item_name} from Google Secret Manager" unless $?.success? diff --git a/lib/kamal/secrets/adapters/last_pass.rb b/lib/kamal/secrets/adapters/last_pass.rb index 2f95148b..be10d0d7 100644 --- a/lib/kamal/secrets/adapters/last_pass.rb +++ b/lib/kamal/secrets/adapters/last_pass.rb @@ -11,7 +11,8 @@ class Kamal::Secrets::Adapters::LastPass < Kamal::Secrets::Adapters::Base `lpass status --color never`.strip == "Logged in as #{account}." end - def fetch_secrets(secrets, account:, session:) + def fetch_secrets(secrets, from:, account:, session:) + secrets = prefixed_secrets(secrets, from: from) items = `lpass show #{secrets.map(&:shellescape).join(" ")} --json` raise RuntimeError, "Could not read #{secrets} from LastPass" unless $?.success? diff --git a/lib/kamal/secrets/adapters/one_password.rb b/lib/kamal/secrets/adapters/one_password.rb index c7f9d5ab..fe454342 100644 --- a/lib/kamal/secrets/adapters/one_password.rb +++ b/lib/kamal/secrets/adapters/one_password.rb @@ -15,9 +15,9 @@ class Kamal::Secrets::Adapters::OnePassword < Kamal::Secrets::Adapters::Base $?.success? end - def fetch_secrets(secrets, account:, session:) + def fetch_secrets(secrets, from:, account:, session:) {}.tap do |results| - vaults_items_fields(secrets).map do |vault, items| + vaults_items_fields(prefixed_secrets(secrets, from: from)).map do |vault, items| items.each do |item, fields| fields_json = JSON.parse(op_item_get(vault, item, fields, account: account, session: session)) fields_json = [ fields_json ] if fields.one? diff --git a/lib/kamal/secrets/adapters/test.rb b/lib/kamal/secrets/adapters/test.rb index 82577a76..ac48960b 100644 --- a/lib/kamal/secrets/adapters/test.rb +++ b/lib/kamal/secrets/adapters/test.rb @@ -4,8 +4,8 @@ class Kamal::Secrets::Adapters::Test < Kamal::Secrets::Adapters::Base true end - def fetch_secrets(secrets, account:, session:) - secrets.to_h { |secret| [ secret, secret.reverse ] } + def fetch_secrets(secrets, from:, account:, session:) + prefixed_secrets(secrets, from: from).to_h { |secret| [ secret, secret.reverse ] } end def check_dependencies! From 5e2678decebe46d4f04ba17985845165039856dc Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 17 Jan 2025 12:28:59 +0000 Subject: [PATCH 35/43] Ensure external input is shell escaped --- lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb b/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb index 7cbc093e..66afbe70 100644 --- a/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb +++ b/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb @@ -18,7 +18,7 @@ class Kamal::Secrets::Adapters::BitwardenSecretsManager < Kamal::Secrets::Adapte {}.tap do |results| if command.nil? secrets.each do |secret_uuid| - secret = run_command("#{GET_COMMAND} #{secret_uuid}") + secret = run_command("#{GET_COMMAND} #{secret_uuid.shellescape}") raise RuntimeError, "Could not read #{secret_uuid} from Bitwarden Secrets Manager" unless $?.success? key, value = parse_secret(secret) results[key] = value @@ -40,7 +40,7 @@ class Kamal::Secrets::Adapters::BitwardenSecretsManager < Kamal::Secrets::Adapte [ LIST_COMMAND, nil ] elsif secrets[0].end_with?(LIST_ALL_FROM_PROJECT_SUFFIX) project = secrets[0].split(LIST_ALL_FROM_PROJECT_SUFFIX).first - [ "#{LIST_COMMAND} #{project}", project ] + [ "#{LIST_COMMAND} #{project.shellescape}", project ] end end end From 10dafc058aff4b635dd816ec9d4660d7b02aeaa9 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 17 Jan 2025 12:31:03 +0000 Subject: [PATCH 36/43] Extract secrets_get_flags --- lib/kamal/secrets/adapters/doppler.rb | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/kamal/secrets/adapters/doppler.rb b/lib/kamal/secrets/adapters/doppler.rb index 40eeeab5..90b2c63b 100644 --- a/lib/kamal/secrets/adapters/doppler.rb +++ b/lib/kamal/secrets/adapters/doppler.rb @@ -18,8 +18,19 @@ class Kamal::Secrets::Adapters::Doppler < Kamal::Secrets::Adapters::Base def fetch_secrets(secrets, from:, **) secrets = prefixed_secrets(secrets, from: from) - project_and_config_flags = "" + flags = secrets_get_flags(secrets) + secret_names = secrets.collect { |s| s.split("/").last } + + items = `doppler secrets get #{secret_names.map(&:shellescape).join(" ")} --json #{flags}` + raise RuntimeError, "Could not read #{secrets} from Doppler" unless $?.success? + + items = JSON.parse(items) + + items.transform_values { |value| value["computed"] } + end + + def secrets_get_flags(secrets) unless service_token_set? project, config, _ = secrets.first.split("/") @@ -29,15 +40,6 @@ class Kamal::Secrets::Adapters::Doppler < Kamal::Secrets::Adapters::Base project_and_config_flags = "-p #{project.shellescape} -c #{config.shellescape}" end - - secret_names = secrets.collect { |s| s.split("/").last } - - items = `doppler secrets get #{secret_names.map(&:shellescape).join(" ")} --json #{project_and_config_flags}` - raise RuntimeError, "Could not read #{secrets} from Doppler" unless $?.success? - - items = JSON.parse(items) - - items.transform_values { |value| value["computed"] } end def service_token_set? From f9a78f4fcbaa244c27f804a34bb9f0e64305ee9d Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 17 Jan 2025 12:34:38 +0000 Subject: [PATCH 37/43] gcloud login tidy Use unless instead of if !, don't suggest running gcloud auth login, we've just tried that. --- lib/kamal/secrets/adapters/gcp_secret_manager.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/kamal/secrets/adapters/gcp_secret_manager.rb b/lib/kamal/secrets/adapters/gcp_secret_manager.rb index 9f4ed41c..8ce381ff 100644 --- a/lib/kamal/secrets/adapters/gcp_secret_manager.rb +++ b/lib/kamal/secrets/adapters/gcp_secret_manager.rb @@ -18,9 +18,9 @@ class Kamal::Secrets::Adapters::GcpSecretManager < Kamal::Secrets::Adapters::Bas # - "default|my-service-user@example.com" will use the default user, and enable service account impersonation as my-service-user # - "default|my-service-user@example.com,another-service-user@example.com" same as above, but with an impersonation delegation chain - if !logged_in? + unless logged_in? `gcloud auth login` - raise RuntimeError, "gcloud is not authenticated, please run `gcloud auth login`" if !logged_in? + raise RuntimeError, "could not login to gcloud" unless logged_in? end nil From 2bd716ece40ef552f2da52aa20888553e23aaec7 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 17 Jan 2025 12:37:12 +0000 Subject: [PATCH 38/43] Drop the TestOptionalAccount adapter It's included in the gem lib which is best to avoid and we can infer that it works account optional adapters. --- lib/kamal/secrets/adapters/test_optional_account.rb | 5 ----- test/cli/secrets_test.rb | 6 ------ 2 files changed, 11 deletions(-) delete mode 100644 lib/kamal/secrets/adapters/test_optional_account.rb diff --git a/lib/kamal/secrets/adapters/test_optional_account.rb b/lib/kamal/secrets/adapters/test_optional_account.rb deleted file mode 100644 index 3a252e68..00000000 --- a/lib/kamal/secrets/adapters/test_optional_account.rb +++ /dev/null @@ -1,5 +0,0 @@ -class Kamal::Secrets::Adapters::TestOptionalAccount < Kamal::Secrets::Adapters::Test - def requires_account? - false - end -end diff --git a/test/cli/secrets_test.rb b/test/cli/secrets_test.rb index bd412862..74f309f7 100644 --- a/test/cli/secrets_test.rb +++ b/test/cli/secrets_test.rb @@ -13,12 +13,6 @@ class CliSecretsTest < CliTestCase run_command("fetch", "foo", "bar", "baz", "--adapter", "test") end - test "fetch without required --account" do - assert_equal \ - "\\{\\\"foo\\\":\\\"oof\\\",\\\"bar\\\":\\\"rab\\\",\\\"baz\\\":\\\"zab\\\"\\}", - run_command("fetch", "foo", "bar", "baz", "--adapter", "test_optional_account") - end - test "extract" do assert_equal "oof", run_command("extract", "foo", "{\"foo\":\"oof\", \"bar\":\"rab\", \"baz\":\"zab\"}") end From 9f1688da7aeab35fa6a760119b68ebfad3ed4b78 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 17 Jan 2025 12:52:23 +0000 Subject: [PATCH 39/43] Fix test --- test/secrets/gcp_secret_manager_adapter_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/secrets/gcp_secret_manager_adapter_test.rb b/test/secrets/gcp_secret_manager_adapter_test.rb index 341f42e9..682db1f4 100644 --- a/test/secrets/gcp_secret_manager_adapter_test.rb +++ b/test/secrets/gcp_secret_manager_adapter_test.rb @@ -23,7 +23,7 @@ class GcpSecretManagerAdapterTest < SecretAdapterTestCase JSON.parse(shellunescape(run_command("fetch", "mypassword"))) end - assert_match(/not authenticated/, error.message) + assert_match(/could not login to gcloud/, error.message) end test "fetch with from" do From 4ffa77220191eee3f89d5f2876436c984b79f7de Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 17 Jan 2025 12:50:03 +0000 Subject: [PATCH 40/43] Don't boot proxy twice when setting up --- lib/kamal/cli/main.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/kamal/cli/main.rb b/lib/kamal/cli/main.rb index b06a3447..4d55e890 100644 --- a/lib/kamal/cli/main.rb +++ b/lib/kamal/cli/main.rb @@ -13,14 +13,14 @@ class Kamal::Cli::Main < Kamal::Cli::Base invoke "kamal:cli:proxy:boot", [], invoke_options invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options - deploy + deploy(boot_proxy: false) end end end desc "deploy", "Deploy app to servers" option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push" - def deploy + def deploy(boot_proxy: true) runtime = print_runtime do invoke_options = deploy_options @@ -39,7 +39,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base run_hook "pre-deploy", secrets: true say "Ensure kamal-proxy is running...", :magenta - invoke "kamal:cli:proxy:boot", [], invoke_options + invoke "kamal:cli:proxy:boot", [], invoke_options if boot_proxy say "Detect stale containers...", :magenta invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true) From 1db44c402c58ac864d38d773748a4b6b4af3c890 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 17 Jan 2025 14:46:27 +0000 Subject: [PATCH 41/43] Boot accessories after pre-deploy hook That allows you to set proxy config in the hook before booting the proxy. --- lib/kamal/cli/main.rb | 12 +++++------- test/cli/main_test.rb | 4 +--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/lib/kamal/cli/main.rb b/lib/kamal/cli/main.rb index 4d55e890..2fae36e8 100644 --- a/lib/kamal/cli/main.rb +++ b/lib/kamal/cli/main.rb @@ -9,18 +9,14 @@ class Kamal::Cli::Main < Kamal::Cli::Base say "Ensure Docker is installed...", :magenta invoke "kamal:cli:server:bootstrap", [], invoke_options - say "Ensure kamal-proxy is running...", :magenta - invoke "kamal:cli:proxy:boot", [], invoke_options - - invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options - deploy(boot_proxy: false) + deploy(boot_accessories: true) end end end desc "deploy", "Deploy app to servers" option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push" - def deploy(boot_proxy: true) + def deploy(boot_accessories: false) runtime = print_runtime do invoke_options = deploy_options @@ -39,7 +35,9 @@ class Kamal::Cli::Main < Kamal::Cli::Base run_hook "pre-deploy", secrets: true say "Ensure kamal-proxy is running...", :magenta - invoke "kamal:cli:proxy:boot", [], invoke_options if boot_proxy + invoke "kamal:cli:proxy:boot", [], invoke_options + + invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options if boot_accessories say "Detect stale containers...", :magenta invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true) diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index 8f2ba9fe..e901c3ba 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -8,9 +8,7 @@ class CliMainTest < CliTestCase invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options) - Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:boot", [], invoke_options) - Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options) - Kamal::Cli::Main.any_instance.expects(:deploy) + Kamal::Cli::Main.any_instance.expects(:deploy).with(boot_accessories: true) run_command("setup").tap do |output| assert_match /Ensure Docker is installed.../, output From dc9a95db2cafeef93f176e9c51b8cc752aa9703a Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 17 Jan 2025 14:58:27 +0000 Subject: [PATCH 42/43] Check for docker locally before registry login We were checking before `kamal build push`, but not `kamal registry login`. Since `kamal registry login` is called first by a deploy we don't get the nice error message. --- lib/kamal/cli.rb | 1 + lib/kamal/cli/base.rb | 14 ++++++++++++++ lib/kamal/cli/build.rb | 16 +--------------- lib/kamal/cli/registry.rb | 2 ++ lib/kamal/commands/base.rb | 14 ++++++++++++++ lib/kamal/commands/builder.rb | 20 -------------------- test/cli/build_test.rb | 2 +- test/cli/registry_test.rb | 10 ++++++++++ 8 files changed, 43 insertions(+), 36 deletions(-) diff --git a/lib/kamal/cli.rb b/lib/kamal/cli.rb index dc35c403..769b4a17 100644 --- a/lib/kamal/cli.rb +++ b/lib/kamal/cli.rb @@ -2,6 +2,7 @@ module Kamal::Cli class BootError < StandardError; end class HookError < StandardError; end class LockError < StandardError; end + class DependencyError < StandardError; end end # SSHKit uses instance eval, so we need a global const for ergonomics diff --git a/lib/kamal/cli/base.rb b/lib/kamal/cli/base.rb index 74f75ffc..8dde4752 100644 --- a/lib/kamal/cli/base.rb +++ b/lib/kamal/cli/base.rb @@ -195,5 +195,19 @@ module Kamal::Cli ENV.clear ENV.update(current_env) end + + def ensure_docker_installed + run_locally do + begin + execute *KAMAL.builder.ensure_docker_installed + rescue SSHKit::Command::Failed => e + error = e.message =~ /command not found/ ? + "Docker is not installed locally" : + "Docker buildx plugin is not installed locally" + + raise DependencyError, error + end + end + end end end diff --git a/lib/kamal/cli/build.rb b/lib/kamal/cli/build.rb index 53ecb0bb..8897e2ae 100644 --- a/lib/kamal/cli/build.rb +++ b/lib/kamal/cli/build.rb @@ -13,7 +13,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base def push cli = self - verify_local_dependencies + ensure_docker_installed run_hook "pre-build" uncommitted_changes = Kamal::Git.uncommitted_changes @@ -109,20 +109,6 @@ class Kamal::Cli::Build < Kamal::Cli::Base end private - def verify_local_dependencies - run_locally do - begin - execute *KAMAL.builder.ensure_local_dependencies_installed - rescue SSHKit::Command::Failed => e - build_error = e.message =~ /command not found/ ? - "Docker is not installed locally" : - "Docker buildx plugin is not installed locally" - - raise BuildError, build_error - end - end - end - def connect_to_remote_host(remote_host) remote_uri = URI.parse(remote_host) if remote_uri.scheme == "ssh" diff --git a/lib/kamal/cli/registry.rb b/lib/kamal/cli/registry.rb index 9d5d9d93..2fbdba1d 100644 --- a/lib/kamal/cli/registry.rb +++ b/lib/kamal/cli/registry.rb @@ -3,6 +3,8 @@ class Kamal::Cli::Registry < Kamal::Cli::Base option :skip_local, aliases: "-L", type: :boolean, default: false, desc: "Skip local login" option :skip_remote, aliases: "-R", type: :boolean, default: false, desc: "Skip remote login" def login + ensure_docker_installed + run_locally { execute *KAMAL.registry.login } unless options[:skip_local] on(KAMAL.hosts) { execute *KAMAL.registry.login } unless options[:skip_remote] end diff --git a/lib/kamal/commands/base.rb b/lib/kamal/commands/base.rb index 6d3f71ec..535d17c0 100644 --- a/lib/kamal/commands/base.rb +++ b/lib/kamal/commands/base.rb @@ -34,6 +34,12 @@ module Kamal::Commands [ :rm, path ] end + def ensure_docker_installed + combine \ + ensure_local_docker_installed, + ensure_local_buildx_installed + end + private def combine(*commands, by: "&&") commands @@ -104,5 +110,13 @@ module Kamal::Commands " -i #{key}" end end + + def ensure_local_docker_installed + docker "--version" + end + + def ensure_local_buildx_installed + docker :buildx, "version" + end end end diff --git a/lib/kamal/commands/builder.rb b/lib/kamal/commands/builder.rb index cd2980fb..a426b0ef 100644 --- a/lib/kamal/commands/builder.rb +++ b/lib/kamal/commands/builder.rb @@ -33,24 +33,4 @@ class Kamal::Commands::Builder < Kamal::Commands::Base def hybrid @hybrid ||= Kamal::Commands::Builder::Hybrid.new(config) end - - - def ensure_local_dependencies_installed - if name.native? - ensure_local_docker_installed - else - combine \ - ensure_local_docker_installed, - ensure_local_buildx_installed - end - end - - private - def ensure_local_docker_installed - docker "--version" - end - - def ensure_local_buildx_installed - docker :buildx, "version" - end end diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb index 88f00743..60dfe036 100644 --- a/test/cli/build_test.rb +++ b/test/cli/build_test.rb @@ -155,7 +155,7 @@ class CliBuildTest < CliTestCase .raises(SSHKit::Command::Failed.new("no buildx")) Kamal::Commands::Builder.any_instance.stubs(:native_and_local?).returns(false) - assert_raises(Kamal::Cli::Build::BuildError) { run_command("push") } + assert_raises(Kamal::Cli::DependencyError) { run_command("push") } end test "push pre-build hook failure" do diff --git a/test/cli/registry_test.rb b/test/cli/registry_test.rb index c5423fe7..e89a15e4 100644 --- a/test/cli/registry_test.rb +++ b/test/cli/registry_test.rb @@ -43,6 +43,16 @@ class CliRegistryTest < CliTestCase end end + test "login with no docker" do + stub_setup + SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with(:docker, "--version", "&&", :docker, :buildx, "version") + .raises(SSHKit::Command::Failed.new("command not found")) + + assert_raises(Kamal::Cli::DependencyError) { run_command("login") } + end + + private def run_command(*command) stdouted { Kamal::Cli::Registry.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) } From a5b9c69838211f2603f59a94a82a1c190fe303af Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 17 Jan 2025 13:00:46 +0000 Subject: [PATCH 43/43] Update to Ruby 3.4 from the preview version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ef7d0ae..dcd1b0ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - "3.1" - "3.2" - "3.3" - - "3.4.0-preview2" + - "3.4" gemfile: - Gemfile - gemfiles/rails_edge.gemfile