diff --git a/Gemfile.lock b/Gemfile.lock index b3946bdc..8a8a1097 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,7 +6,7 @@ PATH base64 (~> 0.2) bcrypt_pbkdf (~> 1.0) concurrent-ruby (~> 1.2) - dotenv (~> 2.8) + dotenv (~> 3.1) ed25519 (~> 1.2) net-ssh (~> 7.0) sshkit (>= 1.23.0, < 2.0) @@ -16,9 +16,9 @@ PATH GEM remote: https://rubygems.org/ specs: - actionpack (7.1.2) - actionview (= 7.1.2) - activesupport (= 7.1.2) + actionpack (7.1.3.4) + actionview (= 7.1.3.4) + activesupport (= 7.1.3.4) nokogiri (>= 1.8.5) racc rack (>= 2.2.4) @@ -26,13 +26,13 @@ GEM rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - actionview (7.1.2) - activesupport (= 7.1.2) + actionview (7.1.3.4) + activesupport (= 7.1.3.4) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activesupport (7.1.2) + activesupport (7.1.3.4) base64 bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) @@ -44,54 +44,55 @@ GEM tzinfo (~> 2.0) ast (2.4.2) base64 (0.2.0) - bcrypt_pbkdf (1.1.0) - bigdecimal (3.1.5) - builder (3.2.4) - concurrent-ruby (1.2.2) + bcrypt_pbkdf (1.1.1) + bcrypt_pbkdf (1.1.1-arm64-darwin) + bcrypt_pbkdf (1.1.1-x86_64-darwin) + bigdecimal (3.1.8) + builder (3.3.0) + concurrent-ruby (1.3.3) connection_pool (2.4.1) crass (1.0.6) - debug (1.9.1) + debug (1.9.2) irb (~> 1.10) reline (>= 0.3.8) - dotenv (2.8.1) - drb (2.2.0) - ruby2_keywords + dotenv (3.1.2) + drb (2.2.1) ed25519 (1.3.0) - erubi (1.12.0) - i18n (1.14.1) + erubi (1.13.0) + i18n (1.14.5) concurrent-ruby (~> 1.0) - io-console (0.7.1) - irb (1.11.0) - rdoc - reline (>= 0.3.8) - json (2.7.1) + io-console (0.7.2) + irb (1.14.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + json (2.7.2) language_server-protocol (3.17.0.3) loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) - minitest (5.20.0) - mocha (2.1.0) + minitest (5.24.1) + mocha (2.4.5) ruby2_keywords (>= 0.0.5) mutex_m (0.2.0) net-scp (4.0.0) net-ssh (>= 2.6.5, < 8.0.0) net-sftp (4.0.0) net-ssh (>= 5.0.0, < 8.0.0) - net-ssh (7.2.1) - nokogiri (1.16.0-arm64-darwin) + net-ssh (7.2.3) + nokogiri (1.16.7-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.0-x86_64-darwin) + nokogiri (1.16.7-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.0-x86_64-linux) + nokogiri (1.16.7-x86_64-linux) racc (~> 1.4) - parallel (1.24.0) - parser (3.3.0.5) + parallel (1.25.1) + parser (3.3.4.0) ast (~> 2.4.1) racc psych (5.1.2) stringio - racc (1.7.3) - rack (3.0.8) + racc (1.8.1) + rack (3.1.7) rack-session (2.0.0) rack (>= 3.0.0) rack-test (2.1.0) @@ -106,42 +107,43 @@ GEM rails-html-sanitizer (1.6.0) loofah (~> 2.21) nokogiri (~> 1.14) - railties (7.1.2) - actionpack (= 7.1.2) - activesupport (= 7.1.2) + railties (7.1.3.4) + actionpack (= 7.1.3.4) + activesupport (= 7.1.3.4) irb rackup (>= 1.0.0) rake (>= 12.2) thor (~> 1.0, >= 1.2.2) zeitwerk (~> 2.6) rainbow (3.1.1) - rake (13.1.0) - rdoc (6.6.2) + rake (13.2.1) + rdoc (6.7.0) psych (>= 4.0.0) - regexp_parser (2.9.0) - reline (0.4.2) + regexp_parser (2.9.2) + reline (0.5.9) io-console (~> 0.5) - rexml (3.2.6) - rubocop (1.62.1) + rexml (3.3.4) + strscan + rubocop (1.65.1) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) + regexp_parser (>= 2.4, < 3.0) rexml (>= 3.2.5, < 4.0) rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.2) - parser (>= 3.3.0.4) - rubocop-minitest (0.35.0) + rubocop-ast (1.32.0) + parser (>= 3.3.1.0) + rubocop-minitest (0.35.1) rubocop (>= 1.61, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) - rubocop-performance (1.20.2) + rubocop-performance (1.21.1) rubocop (>= 1.48.1, < 2.0) - rubocop-ast (>= 1.30.0, < 2.0) - rubocop-rails (2.24.0) + rubocop-ast (>= 1.31.1, < 2.0) + rubocop-rails (2.25.1) activesupport (>= 4.2.0) rack (>= 1.1) rubocop (>= 1.33.0, < 2.0) @@ -158,13 +160,14 @@ GEM net-scp (>= 1.1.2) net-sftp (>= 2.1.2) net-ssh (>= 2.8.0) - stringio (3.1.0) - thor (1.3.0) + stringio (3.1.1) + strscan (3.1.0) + thor (1.3.1) tzinfo (2.0.6) concurrent-ruby (~> 1.0) unicode-display_width (2.5.0) webrick (1.8.1) - zeitwerk (2.6.12) + zeitwerk (2.6.17) PLATFORMS arm64-darwin diff --git a/kamal.gemspec b/kamal.gemspec index ff499f4e..0dfab60b 100644 --- a/kamal.gemspec +++ b/kamal.gemspec @@ -15,7 +15,7 @@ Gem::Specification.new do |spec| spec.add_dependency "sshkit", ">= 1.23.0", "< 2.0" spec.add_dependency "net-ssh", "~> 7.0" spec.add_dependency "thor", "~> 1.3" - spec.add_dependency "dotenv", "~> 2.8" + spec.add_dependency "dotenv", "~> 3.1" spec.add_dependency "zeitwerk", "~> 2.5" spec.add_dependency "ed25519", "~> 1.2" spec.add_dependency "bcrypt_pbkdf", "~> 1.0" diff --git a/lib/kamal.rb b/lib/kamal.rb index 2da2bbf2..6625a9e4 100644 --- a/lib/kamal.rb +++ b/lib/kamal.rb @@ -5,8 +5,10 @@ end require "active_support" require "zeitwerk" require "yaml" +require "tmpdir" +require "pathname" loader = Zeitwerk::Loader.for_gem loader.ignore(File.join(__dir__, "kamal", "sshkit_with_ext.rb")) loader.setup -loader.eager_load # We need all commands loaded. +loader.eager_load_namespace(Kamal::Cli) # We need all commands loaded. diff --git a/lib/kamal/cli/accessory.rb b/lib/kamal/cli/accessory.rb index b3ff10f5..2bf9a786 100644 --- a/lib/kamal/cli/accessory.rb +++ b/lib/kamal/cli/accessory.rb @@ -12,6 +12,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base on(hosts) do execute *KAMAL.registry.login if login execute *KAMAL.auditor.record("Booted #{name} accessory"), verbosity: :debug + execute *accessory.ensure_env_directory + upload! accessory.secrets_io, accessory.secrets_path, mode: "0600" execute *accessory.run end end diff --git a/lib/kamal/cli/app/boot.rb b/lib/kamal/cli/app/boot.rb index b78763ce..df3e6925 100644 --- a/lib/kamal/cli/app/boot.rb +++ b/lib/kamal/cli/app/boot.rb @@ -1,6 +1,6 @@ class Kamal::Cli::App::Boot attr_reader :host, :role, :version, :barrier, :sshkit - delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, to: :sshkit + delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, :upload!, to: :sshkit delegate :uses_cord?, :assets?, :running_traefik?, to: :role def initialize(host, role, sshkit, version, barrier) @@ -48,7 +48,11 @@ class Kamal::Cli::App::Boot execute *app.tie_cord(role.cord_host_file) if uses_cord? hostname = "#{host.to_s[0...51].gsub(/\.+$/, '')}-#{SecureRandom.hex(6)}" + + execute *app.ensure_env_directory + upload! role.secrets_io(host), role.secrets_path, mode: "0600" execute *app.run(hostname: hostname) + Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) } end @@ -88,8 +92,12 @@ class Kamal::Cli::App::Boot def close_barrier if barrier.close info "First #{KAMAL.primary_role} container is unhealthy on #{host}, not booting any other roles" - error capture_with_info(*app.logs(version: version)) - error capture_with_info(*app.container_health_log(version: version)) + begin + error capture_with_info(*app.logs(version: version)) + error capture_with_info(*app.container_health_log(version: version)) + rescue SSHKit::Command::Failed + error "Could not fetch logs for #{version}" + end end end diff --git a/lib/kamal/cli/base.rb b/lib/kamal/cli/base.rb index 8032a4cb..85815506 100644 --- a/lib/kamal/cli/base.rb +++ b/lib/kamal/cli/base.rb @@ -1,5 +1,4 @@ require "thor" -require "dotenv" require "kamal/sshkit_with_ext" module Kamal::Cli @@ -31,53 +30,15 @@ module Kamal::Cli else super end - @original_env = ENV.to_h.dup - load_env - initialize_commander(options_with_subcommand_class_options) + initialize_commander unless KAMAL.configured? end private - def reload_env - reset_env - load_env - end - - def load_env - if destination = options[:destination] - Dotenv.load(".env.#{destination}", ".env") - else - Dotenv.load(".env") - end - end - - def reset_env - replace_env @original_env - end - - def replace_env(env) - ENV.clear - ENV.update(env) - end - - def with_original_env - keeping_current_env do - reset_env - yield - end - end - - def keeping_current_env - current_env = ENV.to_h.dup - yield - ensure - replace_env(current_env) - end - def options_with_subcommand_class_options options.merge(@_initializer.last[:class_options] || {}) end - def initialize_commander(options) + def initialize_commander KAMAL.tap do |commander| if options[:verbose] ENV["VERBOSE"] = "1" # For backtraces via cli/start @@ -112,8 +73,6 @@ module Kamal::Cli if KAMAL.holding_lock? yield else - ensure_run_and_locks_directory - acquire_lock begin @@ -142,6 +101,8 @@ module Kamal::Cli end def acquire_lock + ensure_run_and_locks_directory + raise_if_locked do say "Acquiring the deploy lock...", :magenta on(KAMAL.primary_host) { execute *KAMAL.lock.acquire("Automatic deploy lock", KAMAL.config.version), verbosity: :debug } diff --git a/lib/kamal/cli/build.rb b/lib/kamal/cli/build.rb index 93e1efd9..0347d18c 100644 --- a/lib/kamal/cli/build.rb +++ b/lib/kamal/cli/build.rb @@ -51,7 +51,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base push = KAMAL.builder.push KAMAL.with_verbosity(:debug) do - Dir.chdir(KAMAL.config.builder.build_directory) { execute *push } + Dir.chdir(KAMAL.config.builder.build_directory) { execute *push, env: KAMAL.config.builder.secrets } end end end diff --git a/lib/kamal/cli/env.rb b/lib/kamal/cli/env.rb deleted file mode 100644 index f12174a7..00000000 --- a/lib/kamal/cli/env.rb +++ /dev/null @@ -1,54 +0,0 @@ -require "tempfile" - -class Kamal::Cli::Env < Kamal::Cli::Base - desc "push", "Push the env file to the remote hosts" - def push - with_lock do - on(KAMAL.hosts) do - execute *KAMAL.auditor.record("Pushed env files"), verbosity: :debug - - KAMAL.roles_on(host).each do |role| - execute *KAMAL.app(role: role, host: host).make_env_directory - upload! role.env(host).secrets_io, role.env(host).secrets_file, mode: 400 - end - end - - on(KAMAL.traefik_hosts) do - execute *KAMAL.traefik.make_env_directory - upload! KAMAL.traefik.env.secrets_io, KAMAL.traefik.env.secrets_file, mode: 400 - end - - on(KAMAL.accessory_hosts) do - KAMAL.accessories_on(host).each do |accessory| - accessory_config = KAMAL.config.accessory(accessory) - execute *KAMAL.accessory(accessory).make_env_directory - upload! accessory_config.env.secrets_io, accessory_config.env.secrets_file, mode: 400 - end - end - end - end - - desc "delete", "Delete the env file from the remote hosts" - def delete - with_lock do - on(KAMAL.hosts) do - execute *KAMAL.auditor.record("Deleted env files"), verbosity: :debug - - KAMAL.roles_on(host).each do |role| - execute *KAMAL.app(role: role, host: host).remove_env_file - end - end - - on(KAMAL.traefik_hosts) do - execute *KAMAL.traefik.remove_env_file - end - - on(KAMAL.accessory_hosts) do - KAMAL.accessories_on(host).each do |accessory| - accessory_config = KAMAL.config.accessory(accessory) - execute *KAMAL.accessory(accessory).remove_env_file - end - end - end - end -end diff --git a/lib/kamal/cli/healthcheck/barrier.rb b/lib/kamal/cli/healthcheck/barrier.rb index 0fbfb511..a5db919c 100644 --- a/lib/kamal/cli/healthcheck/barrier.rb +++ b/lib/kamal/cli/healthcheck/barrier.rb @@ -1,3 +1,5 @@ +require "concurrent/ivar" + class Kamal::Cli::Healthcheck::Barrier def initialize @ivar = Concurrent::IVar.new diff --git a/lib/kamal/cli/lock.rb b/lib/kamal/cli/lock.rb index 1e4b52cf..306c8a07 100644 --- a/lib/kamal/cli/lock.rb +++ b/lib/kamal/cli/lock.rb @@ -3,7 +3,6 @@ class Kamal::Cli::Lock < Kamal::Cli::Base def status handle_missing_lock do on(KAMAL.primary_host) do - execute *KAMAL.server.ensure_run_directory puts capture_with_debug(*KAMAL.lock.status) end end @@ -13,9 +12,10 @@ class Kamal::Cli::Lock < Kamal::Cli::Base option :message, aliases: "-m", type: :string, desc: "A lock message", required: true def acquire message = options[:message] + ensure_run_and_locks_directory + raise_if_locked do on(KAMAL.primary_host) do - execute *KAMAL.server.ensure_run_directory execute *KAMAL.lock.acquire(message, KAMAL.config.version), verbosity: :debug end say "Acquired the deploy lock" @@ -26,7 +26,6 @@ class Kamal::Cli::Lock < Kamal::Cli::Base def release handle_missing_lock do on(KAMAL.primary_host) do - execute *KAMAL.server.ensure_run_directory execute *KAMAL.lock.release, verbosity: :debug end say "Released the deploy lock" diff --git a/lib/kamal/cli/main.rb b/lib/kamal/cli/main.rb index 598d6c42..3bd6dc24 100644 --- a/lib/kamal/cli/main.rb +++ b/lib/kamal/cli/main.rb @@ -9,10 +9,6 @@ class Kamal::Cli::Main < Kamal::Cli::Base say "Ensure Docker is installed...", :magenta invoke "kamal:cli:server:bootstrap", [], invoke_options - say "Evaluate and push env files...", :magenta - invoke "kamal:cli:main:envify", [], invoke_options - invoke "kamal:cli:env:push", [], invoke_options - invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options deploy end @@ -37,7 +33,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base end with_lock do - run_hook "pre-deploy" + run_hook "pre-deploy", secrets: true say "Ensure Traefik is running...", :magenta invoke "kamal:cli:traefik:boot", [], invoke_options @@ -52,7 +48,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base end end - run_hook "post-deploy", runtime: runtime.round + run_hook "post-deploy", secrets: true, runtime: runtime.round end desc "redeploy", "Deploy app to servers without bootstrapping servers, starting Traefik, pruning, and registry login" @@ -70,7 +66,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base end with_lock do - run_hook "pre-deploy" + run_hook "pre-deploy", secrets: true say "Detect stale containers...", :magenta invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true) @@ -79,7 +75,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base end end - run_hook "post-deploy", runtime: runtime.round + run_hook "post-deploy", secrets: true, runtime: runtime.round end desc "rollback [VERSION]", "Rollback app to VERSION" @@ -93,7 +89,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base old_version = nil if container_available?(version) - run_hook "pre-deploy" + run_hook "pre-deploy", secrets: true invoke "kamal:cli:app:boot", [], invoke_options.merge(version: version) rolled_back = true @@ -103,7 +99,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base end end - run_hook "post-deploy", runtime: runtime.round if rolled_back + run_hook "post-deploy", secrets: true, runtime: runtime.round if rolled_back end desc "details", "Show details about all containers" @@ -152,9 +148,10 @@ class Kamal::Cli::Main < Kamal::Cli::Base puts "Created configuration file in config/deploy.yml" end - unless (deploy_file = Pathname.new(File.expand_path(".env"))).exist? - FileUtils.cp_r Pathname.new(File.expand_path("templates/template.env", __dir__)), deploy_file - puts "Created .env file" + unless (secrets_file = Pathname.new(File.expand_path(".kamal/secrets"))).exist? + FileUtils.mkdir_p secrets_file.dirname + FileUtils.cp_r Pathname.new(File.expand_path("templates/secrets", __dir__)), secrets_file + puts "Created .kamal/secrets file" end unless (hooks_dir = Pathname.new(File.expand_path(".kamal/hooks"))).exist? @@ -179,31 +176,6 @@ class Kamal::Cli::Main < Kamal::Cli::Base end end - desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)" - option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip .env file push" - def envify - if destination = options[:destination] - env_template_path = ".env.#{destination}.erb" - env_path = ".env.#{destination}" - else - env_template_path = ".env.erb" - env_path = ".env" - end - - if Pathname.new(File.expand_path(env_template_path)).exist? - # Ensure existing env doesn't pollute template evaluation - content = with_original_env { ERB.new(File.read(env_template_path), trim_mode: "-").result } - File.write(env_path, content, perm: 0600) - - unless options[:skip_push] - reload_env - invoke "kamal:cli:env:push", options - end - else - puts "Skipping envify (no #{env_template_path} exist)" - end - end - desc "remove", "Remove Traefik, app, accessories, and registry session from servers" option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question" def remove @@ -231,9 +203,6 @@ class Kamal::Cli::Main < Kamal::Cli::Base desc "build", "Build application image" subcommand "build", Kamal::Cli::Build - desc "env", "Manage environment files" - subcommand "env", Kamal::Cli::Env - desc "lock", "Manage the deploy lock" subcommand "lock", Kamal::Cli::Lock @@ -243,6 +212,9 @@ class Kamal::Cli::Main < Kamal::Cli::Base desc "registry", "Login and -out of the image registry" subcommand "registry", Kamal::Cli::Registry + desc "secrets", "Helpers for extracting secrets" + subcommand "secrets", Kamal::Cli::Secrets + desc "server", "Bootstrap servers with curl and Docker" subcommand "server", Kamal::Cli::Server diff --git a/lib/kamal/cli/secrets.rb b/lib/kamal/cli/secrets.rb new file mode 100644 index 00000000..8b919f9a --- /dev/null +++ b/lib/kamal/cli/secrets.rb @@ -0,0 +1,36 @@ +class Kamal::Cli::Secrets < Kamal::Cli::Base + desc "fetch [SECRETS...]", "Fetch secrets from a vault" + option :adapter, type: :string, aliases: "-a", required: true, desc: "Which vault adapter to use" + option :account, type: :string, required: true, desc: "The account identifier or username" + option :from, type: :string, required: false, desc: "A vault or folder to fetch the secrets from" + option :inline, type: :boolean, required: false, hidden: true + def fetch(*secrets) + results = adapter(options[:adapter]).fetch(secrets, **options.slice(:account, :from).symbolize_keys) + + return_or_puts JSON.dump(results).shellescape, inline: options[:inline] + end + + desc "extract", "Extract a single secret from the results of a fetch call" + option :inline, type: :boolean, required: false, hidden: true + def extract(name, secrets) + parsed_secrets = JSON.parse(secrets) + value = parsed_secrets[name] || parsed_secrets.find { |k, v| k.end_with?("/#{name}") }&.last + + raise "Could not find secret #{name}" if value.nil? + + return_or_puts value, inline: options[:inline] + end + + private + def adapter(adapter) + Kamal::Secrets::Adapters.lookup(adapter) + end + + def return_or_puts(value, inline: nil) + if inline + value + else + puts value + end + end +end diff --git a/lib/kamal/cli/templates/secrets b/lib/kamal/cli/templates/secrets new file mode 100644 index 00000000..91f4f239 --- /dev/null +++ b/lib/kamal/cli/templates/secrets @@ -0,0 +1,16 @@ +# WARNING: Avoid adding secrets directly to this file +# If you must, then add `.kamal/secrets*` to your .gitignore file + +# Option 1: Read secrets from the environment +KAMAL_REGISTRY_PASSWORD=$KAMAL_REGISTRY_PASSWORD + +# Option 2: Read secrets via a command +# RAILS_MASTER_KEY=$(cat config/master.key) + +# Option 3: Read secrets via kamal secrets helpers +# These will handle logging in and fetching the secrets in as few calls as possible +# There are adapters for 1Password, LastPass + Bitwarden +# +# SECRETS=$(kamal secrets fetch --adapter 1password --account my-account --from MyVault/MyItem KAMAL_REGISTRY_PASSWORD RAILS_MASTER_KEY) +# KAMAL_REGISTRY_PASSWORD=$(kamal secrets extract KAMAL_REGISTRY_PASSWORD $SECRETS) +# RAILS_MASTER_KEY=$(kamal secrets extract RAILS_MASTER_KEY $SECRETS) diff --git a/lib/kamal/cli/templates/template.env b/lib/kamal/cli/templates/template.env deleted file mode 100644 index 89411448..00000000 --- a/lib/kamal/cli/templates/template.env +++ /dev/null @@ -1,2 +0,0 @@ -KAMAL_REGISTRY_PASSWORD=change-this -RAILS_MASTER_KEY=another-env diff --git a/lib/kamal/cli/traefik.rb b/lib/kamal/cli/traefik.rb index a8bd2126..41ffbc04 100644 --- a/lib/kamal/cli/traefik.rb +++ b/lib/kamal/cli/traefik.rb @@ -4,6 +4,8 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base with_lock do on(KAMAL.traefik_hosts) do execute *KAMAL.registry.login + execute *KAMAL.traefik.ensure_env_directory + upload! KAMAL.traefik.secrets_io, KAMAL.traefik.secrets_path, mode: "0600" execute *KAMAL.traefik.start_or_run end end diff --git a/lib/kamal/commander.rb b/lib/kamal/commander.rb index ae98e0f8..11914a67 100644 --- a/lib/kamal/commander.rb +++ b/lib/kamal/commander.rb @@ -1,5 +1,6 @@ require "active_support/core_ext/enumerable" require "active_support/core_ext/module/delegation" +require "active_support/core_ext/object/blank" class Kamal::Commander attr_accessor :verbosity, :holding_lock, :connected @@ -23,6 +24,10 @@ class Kamal::Commander @config, @config_kwargs = nil, kwargs end + def configured? + @config || @config_kwargs + end + attr_reader :specific_roles, :specific_hosts def specific_primary! diff --git a/lib/kamal/commands/accessory.rb b/lib/kamal/commands/accessory.rb index 23377ab5..f3b676d1 100644 --- a/lib/kamal/commands/accessory.rb +++ b/lib/kamal/commands/accessory.rb @@ -1,7 +1,9 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base attr_reader :accessory_config delegate :service_name, :image, :hosts, :port, :files, :directories, :cmd, - :publish_args, :env_args, :volume_args, :label_args, :option_args, to: :accessory_config + :publish_args, :env_args, :volume_args, :label_args, :option_args, + :secrets_io, :secrets_path, :env_directory, + to: :accessory_config def initialize(config, name:) super(config) @@ -98,12 +100,8 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base docker :image, :rm, "--force", image end - def make_env_directory - make_directory accessory_config.env.secrets_directory - end - - def remove_env_file - [ :rm, "-f", accessory_config.env.secrets_file ] + def ensure_env_directory + make_directory env_directory end private diff --git a/lib/kamal/commands/app.rb b/lib/kamal/commands/app.rb index 37fa86ab..f1991e48 100644 --- a/lib/kamal/commands/app.rb +++ b/lib/kamal/commands/app.rb @@ -69,16 +69,10 @@ class Kamal::Commands::App < Kamal::Commands::Base extract_version_from_name end - - def make_env_directory - make_directory role.env(host).secrets_directory + def ensure_env_directory + make_directory role.env_directory end - def remove_env_file - [ :rm, "-f", role.env(host).secrets_file ] - end - - private def container_name(version = nil) [ role.container_prefix, version || config.version ].compact.join("-") diff --git a/lib/kamal/commands/app/assets.rb b/lib/kamal/commands/app/assets.rb index 26ca6e0c..9841f4fb 100644 --- a/lib/kamal/commands/app/assets.rb +++ b/lib/kamal/commands/app/assets.rb @@ -5,7 +5,7 @@ module Kamal::Commands::App::Assets combine \ make_directory(role.asset_extracted_path), [ *docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true" ], - docker(:run, "--name", asset_container, "--detach", "--rm", config.absolute_image, "sleep 1000000"), + docker(:run, "--name", asset_container, "--detach", "--rm", "--entrypoint", "sleep", config.absolute_image, "1000000"), docker(:cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_path), docker(:stop, "-t 1", asset_container), by: "&&" diff --git a/lib/kamal/commands/auditor.rb b/lib/kamal/commands/auditor.rb index f0c0850d..9846d8e2 100644 --- a/lib/kamal/commands/auditor.rb +++ b/lib/kamal/commands/auditor.rb @@ -8,9 +8,12 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base # Runs remotely def record(line, **details) - append \ - [ :echo, audit_tags(**details).except(:version, :service_version, :service).to_s, line ], - audit_log_file + combine \ + [ :mkdir, "-p", config.run_directory ], + append( + [ :echo, audit_tags(**details).except(:version, :service_version, :service).to_s, line ], + audit_log_file + ) end def reveal diff --git a/lib/kamal/commands/base.rb b/lib/kamal/commands/base.rb index 59928d95..34081744 100644 --- a/lib/kamal/commands/base.rb +++ b/lib/kamal/commands/base.rb @@ -37,6 +37,10 @@ module Kamal::Commands [ :rm, "-r", path ] end + def remove_file(path) + [ :rm, path ] + end + private def combine(*commands, by: "&&") commands diff --git a/lib/kamal/commands/builder/base.rb b/lib/kamal/commands/builder/base.rb index a0454818..4bcfa206 100644 --- a/lib/kamal/commands/builder/base.rb +++ b/lib/kamal/commands/builder/base.rb @@ -79,7 +79,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base end def build_secrets - argumentize "--secret", secrets.collect { |secret| [ "id", secret ] } + argumentize "--secret", secrets.keys.collect { |secret| [ "id", secret ] } end def build_dockerfile diff --git a/lib/kamal/commands/hook.rb b/lib/kamal/commands/hook.rb index 66fe8b8c..eb710d5e 100644 --- a/lib/kamal/commands/hook.rb +++ b/lib/kamal/commands/hook.rb @@ -1,6 +1,9 @@ class Kamal::Commands::Hook < Kamal::Commands::Base - def run(hook, **details) - [ hook_file(hook), env: tags(**details).env ] + def run(hook, secrets: false, **details) + env = tags(**details).env + env.merge!(config.secrets.to_h) if secrets + + [ hook_file(hook), env: env ] end def hook_exists?(hook) diff --git a/lib/kamal/commands/traefik.rb b/lib/kamal/commands/traefik.rb index 07e0e6ea..964ef3eb 100644 --- a/lib/kamal/commands/traefik.rb +++ b/lib/kamal/commands/traefik.rb @@ -1,6 +1,6 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base delegate :argumentize, :optionize, to: Kamal::Utils - delegate :port, :publish?, :labels, :env, :image, :options, :args, to: :"config.traefik" + delegate :port, :publish?, :labels, :env, :image, :options, :args, :env_args, :secrets_io, :env_directory, :secrets_path, to: :"config.traefik" def run docker :run, "--name traefik", @@ -54,12 +54,8 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=Traefik" end - def make_env_directory - make_directory(env.secrets_directory) - end - - def remove_env_file - [ :rm, "-f", env.secrets_file ] + def ensure_env_directory + make_directory env_directory end private @@ -71,10 +67,6 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base argumentize "--label", labels end - def env_args - env.args - end - def docker_options_args optionize(options) end diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index af3754eb..2758d15a 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -2,7 +2,6 @@ require "active_support/ordered_options" require "active_support/core_ext/string/inquiry" require "active_support/core_ext/module/delegation" require "active_support/core_ext/hash/keys" -require "pathname" require "erb" require "net/ssh/proxy/jump" @@ -10,7 +9,7 @@ class Kamal::Configuration delegate :service, :image, :labels, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true delegate :argumentize, :optionize, to: Kamal::Utils - attr_reader :destination, :raw_config + attr_reader :destination, :raw_config, :secrets attr_reader :accessories, :aliases, :boot, :builder, :env, :healthcheck, :logging, :traefik, :servers, :ssh, :sshkit, :registry include Validation @@ -57,7 +56,7 @@ class Kamal::Configuration @aliases = @raw_config.aliases&.keys&.to_h { |name| [ name, Alias.new(name, config: self) ] } || {} @boot = Boot.new(config: self) @builder = Builder.new(config: self) - @env = Env.new(config: @raw_config.env || {}) + @env = Env.new(config: @raw_config.env || {}, secrets: secrets) @healthcheck = Healthcheck.new(healthcheck_config: @raw_config.healthcheck) @logging = Logging.new(logging_config: @raw_config.logging) @@ -65,6 +64,8 @@ class Kamal::Configuration @ssh = Ssh.new(config: self) @sshkit = Sshkit.new(config: self) + @secrets = Kamal::Secrets.new(destination: destination) + ensure_destination_if_required ensure_required_keys_present ensure_valid_kamal_version @@ -218,13 +219,13 @@ class Kamal::Configuration end - def host_env_directory + def env_directory File.join(run_directory, "env") end def env_tags @env_tags ||= if (tags = raw_config.env["tags"]) - tags.collect { |name, config| Env::Tag.new(name, config: config) } + tags.collect { |name, config| Env::Tag.new(name, config: config, secrets: secrets) } else [] end @@ -254,6 +255,10 @@ class Kamal::Configuration }.compact end + def secrets + @secrets ||= Kamal::Secrets.new(destination: destination) + end + private # Will raise ArgumentError if any required config keys are missing def ensure_destination_if_required diff --git a/lib/kamal/configuration/accessory.rb b/lib/kamal/configuration/accessory.rb index 07f40b56..57489f17 100644 --- a/lib/kamal/configuration/accessory.rb +++ b/lib/kamal/configuration/accessory.rb @@ -16,7 +16,7 @@ class Kamal::Configuration::Accessory @env = Kamal::Configuration::Env.new \ config: accessory_config.fetch("env", {}), - secrets_file: File.join(config.host_env_directory, "accessories", "#{service_name}.env"), + secrets: config.secrets, context: "accessories/#{name}/env" end @@ -51,7 +51,19 @@ class Kamal::Configuration::Accessory end def env_args - env.args + [ *env.clear_args, *argumentize("--env-file", secrets_path) ] + end + + def env_directory + File.join(config.env_directory, "accessories") + end + + def secrets_io + env.secrets_io + end + + def secrets_path + File.join(config.env_directory, "accessories", "#{service_name}.env") end def files diff --git a/lib/kamal/configuration/builder.rb b/lib/kamal/configuration/builder.rb index 3efd62f5..71a017a7 100644 --- a/lib/kamal/configuration/builder.rb +++ b/lib/kamal/configuration/builder.rb @@ -66,7 +66,7 @@ class Kamal::Configuration::Builder end def secrets - builder_config["secrets"] || [] + (builder_config["secrets"] || []).to_h { |key| [ key, config.secrets[key] ] } end def dockerfile diff --git a/lib/kamal/configuration/env.rb b/lib/kamal/configuration/env.rb index 1c0fb1e9..8e52d9e4 100644 --- a/lib/kamal/configuration/env.rb +++ b/lib/kamal/configuration/env.rb @@ -1,36 +1,29 @@ class Kamal::Configuration::Env include Kamal::Configuration::Validation - attr_reader :secrets_keys, :clear, :secrets_file, :context + attr_reader :context, :secrets + attr_reader :clear, :secret_keys delegate :argumentize, to: Kamal::Utils - def initialize(config:, secrets_file: nil, context: "env") + def initialize(config:, secrets:, context: "env") @clear = config.fetch("clear", config.key?("secret") || config.key?("tags") ? {} : config) - @secrets_keys = config.fetch("secret", []) - @secrets_file = secrets_file + @secrets = secrets + @secret_keys = config.fetch("secret", []) @context = context validate! config, context: context, with: Kamal::Configuration::Validator::Env end - def args - [ "--env-file", secrets_file, *argumentize("--env", clear) ] + def clear_args + argumentize("--env", clear) end def secrets_io - StringIO.new(Kamal::EnvFile.new(secrets).to_s) - end - - def secrets - @secrets ||= secrets_keys.to_h { |key| [ key, ENV.fetch(key) ] } - end - - def secrets_directory - File.dirname(secrets_file) + Kamal::EnvFile.new(secret_keys.to_h { |key| [ key, secrets[key] ] }).to_io end def merge(other) self.class.new \ - config: { "clear" => clear.merge(other.clear), "secret" => secrets_keys | other.secrets_keys }, - secrets_file: secrets_file || other.secrets_file + config: { "clear" => clear.merge(other.clear), "secret" => secret_keys | other.secret_keys }, + secrets: secrets end end diff --git a/lib/kamal/configuration/env/tag.rb b/lib/kamal/configuration/env/tag.rb index c4151202..30160edc 100644 --- a/lib/kamal/configuration/env/tag.rb +++ b/lib/kamal/configuration/env/tag.rb @@ -1,12 +1,13 @@ class Kamal::Configuration::Env::Tag - attr_reader :name, :config + attr_reader :name, :config, :secrets - def initialize(name, config:) + def initialize(name, config:, secrets:) @name = name @config = config + @secrets = secrets end def env - Kamal::Configuration::Env.new(config: config) + Kamal::Configuration::Env.new(config: config, secrets: secrets) end end diff --git a/lib/kamal/configuration/registry.rb b/lib/kamal/configuration/registry.rb index fa0ba04a..763cf976 100644 --- a/lib/kamal/configuration/registry.rb +++ b/lib/kamal/configuration/registry.rb @@ -1,10 +1,11 @@ class Kamal::Configuration::Registry include Kamal::Configuration::Validation - attr_reader :registry_config + attr_reader :registry_config, :secrets def initialize(config:) @registry_config = config.raw_config.registry || {} + @secrets = config.secrets validate! registry_config, with: Kamal::Configuration::Validator::Registry end @@ -23,7 +24,7 @@ class Kamal::Configuration::Registry private def lookup(key) if registry_config[key].is_a?(Array) - ENV.fetch(registry_config[key].first).dup + secrets[registry_config[key].first] else registry_config[key] end diff --git a/lib/kamal/configuration/role.rb b/lib/kamal/configuration/role.rb index e9e520a7..ef651898 100644 --- a/lib/kamal/configuration/role.rb +++ b/lib/kamal/configuration/role.rb @@ -18,7 +18,7 @@ class Kamal::Configuration::Role @specialized_env = Kamal::Configuration::Env.new \ config: specializations.fetch("env", {}), - secrets_file: File.join(config.host_env_directory, "roles", "#{container_prefix}.env"), + secrets: config.secrets, context: "servers/#{name}/env" @specialized_logging = Kamal::Configuration::Logging.new \ @@ -77,7 +77,19 @@ class Kamal::Configuration::Role end def env_args(host) - env(host).args + [ *env(host).clear_args, *argumentize("--env-file", secrets_path) ] + end + + def env_directory + File.join(config.env_directory, "roles") + end + + def secrets_io(host) + env(host).secrets_io + end + + def secrets_path + File.join(config.env_directory, "roles", "#{container_prefix}.env") end def asset_volume_args diff --git a/lib/kamal/configuration/traefik.rb b/lib/kamal/configuration/traefik.rb index c958afdf..45d8bac5 100644 --- a/lib/kamal/configuration/traefik.rb +++ b/lib/kamal/configuration/traefik.rb @@ -1,4 +1,6 @@ class Kamal::Configuration::Traefik + delegate :argumentize, to: Kamal::Utils + DEFAULT_IMAGE = "traefik:v2.10" CONTAINER_PORT = 80 DEFAULT_ARGS = { @@ -34,7 +36,7 @@ class Kamal::Configuration::Traefik def env Kamal::Configuration::Env.new \ config: traefik_config.fetch("env", {}), - secrets_file: File.join(config.host_env_directory, "traefik", "traefik.env"), + secrets: config.secrets, context: "traefik/env" end @@ -57,4 +59,20 @@ class Kamal::Configuration::Traefik def image traefik_config.fetch("image", DEFAULT_IMAGE) end + + def env_args + [ *env.clear_args, *argumentize("--env-file", secrets_path) ] + end + + def env_directory + File.join(config.env_directory, "traefik") + end + + def secrets_io + env.secrets_io + end + + def secrets_path + File.join(config.env_directory, "traefik", "traefik.env") + end end diff --git a/lib/kamal/env_file.rb b/lib/kamal/env_file.rb index 2228be09..6a4a80e3 100644 --- a/lib/kamal/env_file.rb +++ b/lib/kamal/env_file.rb @@ -15,6 +15,10 @@ class Kamal::EnvFile env_file.presence || "\n" end + def to_io + StringIO.new(to_s) + end + alias to_str to_s private diff --git a/lib/kamal/secrets.rb b/lib/kamal/secrets.rb new file mode 100644 index 00000000..c7d4cc03 --- /dev/null +++ b/lib/kamal/secrets.rb @@ -0,0 +1,37 @@ +require "dotenv" + +class Kamal::Secrets + attr_reader :secrets_files + + Kamal::Secrets::Dotenv::InlineCommandSubstitution.install! + + def initialize(destination: nil) + @secrets_files = \ + [ ".kamal/secrets-common", ".kamal/secrets#{(".#{destination}" if destination)}" ].select { |f| File.exist?(f) } + @mutex = Mutex.new + end + + def [](key) + # Fetching secrets may ask the user for input, so ensure only one thread does that + @mutex.synchronize do + secrets.fetch(key) + end + rescue KeyError + if secrets_files + raise Kamal::ConfigurationError, "Secret '#{key}' not found in #{secrets_files.join(", ")}" + else + raise Kamal::ConfigurationError, "Secret '#{key}' not found, no secret files provided" + end + end + + def to_h + secrets + end + + private + def secrets + @secrets ||= secrets_files.inject({}) do |secrets, secrets_file| + secrets.merge!(::Dotenv.parse(secrets_file)) + end + end +end diff --git a/lib/kamal/secrets/adapters.rb b/lib/kamal/secrets/adapters.rb new file mode 100644 index 00000000..439c7208 --- /dev/null +++ b/lib/kamal/secrets/adapters.rb @@ -0,0 +1,14 @@ +require "active_support/core_ext/string/inflections" +module Kamal::Secrets::Adapters + def self.lookup(name) + name = "one_password" if name.downcase == "1password" + name = "last_pass" if name.downcase == "lastpass" + adapter_class(name) + end + + def self.adapter_class(name) + Object.const_get("Kamal::Secrets::Adapters::#{name.camelize}").new + rescue NameError => e + raise RuntimeError, "Unknown secrets adapter: #{name}" + end +end diff --git a/lib/kamal/secrets/adapters/base.rb b/lib/kamal/secrets/adapters/base.rb new file mode 100644 index 00000000..97b2a458 --- /dev/null +++ b/lib/kamal/secrets/adapters/base.rb @@ -0,0 +1,18 @@ +class Kamal::Secrets::Adapters::Base + delegate :optionize, to: Kamal::Utils + + def fetch(secrets, account:, from: nil) + session = login(account) + full_secrets = secrets.map { |secret| [ from, secret ].compact.join("/") } + fetch_secrets(full_secrets, account: account, session: session) + end + + private + def login(...) + raise NotImplementedError + end + + def fetch_secrets(...) + raise NotImplementedError + end +end diff --git a/lib/kamal/secrets/adapters/bitwarden.rb b/lib/kamal/secrets/adapters/bitwarden.rb new file mode 100644 index 00000000..95716997 --- /dev/null +++ b/lib/kamal/secrets/adapters/bitwarden.rb @@ -0,0 +1,64 @@ +class Kamal::Secrets::Adapters::Bitwarden < Kamal::Secrets::Adapters::Base + private + def login(account) + status = run_command("status") + + if status["status"] == "unauthenticated" + run_command("login #{account.shellescape}", raw: true) + status = run_command("status") + end + + if status["status"] == "locked" + session = run_command("unlock --raw", raw: true).presence + status = run_command("status", session: session) + end + + raise RuntimeError, "Failed to login to and unlock Bitwarden" unless status["status"] == "unlocked" + + run_command("sync", session: session, raw: true) + raise RuntimeError, "Failed to sync Bitwarden" unless $?.success? + + session + end + + def fetch_secrets(secrets, account:, session:) + {}.tap do |results| + items_fields(secrets).each do |item, fields| + item_json = run_command("get item #{item.shellescape}", session: session, raw: true) + raise RuntimeError, "Could not read #{secret} from Bitwarden" unless $?.success? + item_json = JSON.parse(item_json) + + if fields.any? + fields.each do |field| + item_field = item_json["fields"].find { |f| f["name"] == field } + raise RuntimeError, "Could not find field #{field} in item #{item} in Bitwarden" unless item_field + value = item_field["value"] + results["#{item}/#{field}"] = value + end + else + results[item] = item_json["login"]["password"] + end + end + end + end + + def items_fields(secrets) + {}.tap do |items| + secrets.each do |secret| + item, field = secret.split("/") + items[item] ||= [] + items[item] << field + end + end + end + + def signedin?(account) + run_command("status")["status"] != "unauthenticated" + end + + def run_command(command, session: nil, raw: false) + full_command = [ *("BW_SESSION=#{session.shellescape}" if session), "bw", command ].join(" ") + result = `#{full_command}`.strip + raw ? result : JSON.parse(result) + end +end diff --git a/lib/kamal/secrets/adapters/last_pass.rb b/lib/kamal/secrets/adapters/last_pass.rb new file mode 100644 index 00000000..16dad150 --- /dev/null +++ b/lib/kamal/secrets/adapters/last_pass.rb @@ -0,0 +1,30 @@ +class Kamal::Secrets::Adapters::LastPass < Kamal::Secrets::Adapters::Base + private + def login(account) + unless loggedin?(account) + `lpass login #{account.shellescape}` + raise RuntimeError, "Failed to login to 1Password" unless $?.success? + end + end + + def loggedin?(account) + `lpass status --color never`.strip == "Logged in as #{account}." + end + + def fetch_secrets(secrets, account:, session:) + items = `lpass show #{secrets.map(&:shellescape).join(" ")} --json` + raise RuntimeError, "Could not read #{secrets} from 1Password" unless $?.success? + + items = JSON.parse(items) + + {}.tap do |results| + items.each do |item| + results[item["fullname"]] = item["password"] + end + + if (missing_items = secrets - results.keys).any? + raise RuntimeError, "Could not find #{missing_items.join(", ")} in LassPass" + end + end + end +end diff --git a/lib/kamal/secrets/adapters/one_password.rb b/lib/kamal/secrets/adapters/one_password.rb new file mode 100644 index 00000000..c7e9b28d --- /dev/null +++ b/lib/kamal/secrets/adapters/one_password.rb @@ -0,0 +1,61 @@ +class Kamal::Secrets::Adapters::OnePassword < Kamal::Secrets::Adapters::Base + delegate :optionize, to: Kamal::Utils + + private + def login(account) + unless loggedin?(account) + `op signin #{to_options(account: account, force: true, raw: true)}`.tap do + raise RuntimeError, "Failed to login to 1Password" unless $?.success? + end + end + end + + def loggedin?(account) + `op account get --account #{account.shellescape} 2> /dev/null` + $?.success? + end + + def fetch_secrets(secrets, account:, session:) + {}.tap do |results| + vaults_items_fields(secrets).map do |vault, items| + items.each do |item, fields| + fields_json = JSON.parse(op_item_get(vault, item, fields, account: account, session: session)) + fields_json = [ fields_json ] if fields.one? + + fields_json.each do |field_json| + # The reference is in the form `op://vault/item/field[/field]` + field = field_json["reference"].delete_prefix("op://").delete_suffix("/password") + results[field] = field_json["value"] + end + end + end + end + end + + def to_options(**options) + optionize(options.compact).join(" ") + end + + def vaults_items_fields(secrets) + {}.tap do |vaults| + secrets.each do |secret| + secret = secret.delete_prefix("op://") + vault, item, *fields = secret.split("/") + fields << "password" if fields.empty? + + vaults[vault] ||= {} + vaults[vault][item] ||= [] + vaults[vault][item] << fields.join(".") + end + end + end + + def op_item_get(vault, item, fields, account:, session:) + labels = fields.map { |field| "label=#{field}" }.join(",") + options = to_options(vault: vault, fields: labels, format: "json", account: account, session: session.presence) + + `op item get #{item.shellescape} #{options}`.tap do + raise RuntimeError, "Could not read #{fields.join(", ")} from #{item} in the #{vault} 1Password vault" unless $?.success? + end + end +end diff --git a/lib/kamal/secrets/adapters/test.rb b/lib/kamal/secrets/adapters/test.rb new file mode 100644 index 00000000..fc0903d9 --- /dev/null +++ b/lib/kamal/secrets/adapters/test.rb @@ -0,0 +1,10 @@ +class Kamal::Secrets::Adapters::Test < Kamal::Secrets::Adapters::Base + private + def login(account) + true + end + + def fetch_secrets(secrets, account:, session:) + secrets.to_h { |secret| [ secret, secret.reverse ] } + end +end diff --git a/lib/kamal/secrets/dotenv/inline_command_substitution.rb b/lib/kamal/secrets/dotenv/inline_command_substitution.rb new file mode 100644 index 00000000..c9ef9879 --- /dev/null +++ b/lib/kamal/secrets/dotenv/inline_command_substitution.rb @@ -0,0 +1,32 @@ +class Kamal::Secrets::Dotenv::InlineCommandSubstitution + class << self + def install! + ::Dotenv::Parser.substitutions.map! { |sub| sub == ::Dotenv::Substitutions::Command ? self : sub } + end + + def call(value, _env, overwrite: false) + # Process interpolated shell commands + value.gsub(Dotenv::Substitutions::Command.singleton_class::INTERPOLATED_SHELL_COMMAND) do |*| + # Eliminate opening and closing parentheses + command = $LAST_MATCH_INFO[:cmd][1..-2] + + if $LAST_MATCH_INFO[:backslash] + # Command is escaped, don't replace it. + $LAST_MATCH_INFO[0][1..] + else + if command =~ /\A\s*kamal\s*secrets\s+/ + # Inline the command + inline_secrets_command(command) + else + # Execute the command and return the value + `#{command}`.chomp + end + end + end + end + + def inline_secrets_command(command) + Kamal::Cli::Main.start(command.shellsplit[1..] + [ "--inline" ]).chomp + end + end +end diff --git a/lib/kamal/sshkit_with_ext.rb b/lib/kamal/sshkit_with_ext.rb index 2d0257a8..ab6795e7 100644 --- a/lib/kamal/sshkit_with_ext.rb +++ b/lib/kamal/sshkit_with_ext.rb @@ -3,6 +3,7 @@ require "sshkit/dsl" require "net/scp" require "active_support/core_ext/hash/deep_merge" require "json" +require "concurrent/atomic/semaphore" class SSHKit::Backend::Abstract def capture_with_info(*args, **kwargs) diff --git a/lib/kamal/utils.rb b/lib/kamal/utils.rb index 46736ba5..266d6a96 100644 --- a/lib/kamal/utils.rb +++ b/lib/kamal/utils.rb @@ -1,3 +1,5 @@ +require "active_support/core_ext/object/try" + module Kamal::Utils extend self @@ -54,6 +56,12 @@ module Kamal::Utils # Escape a value to make it safe for shell use. def escape_shell_value(value) + value.to_s.scan(/[\x00-\x7F]+|[^\x00-\x7F]+/) \ + .map { |part| part.ascii_only? ? escape_ascii_shell_value(part) : part } + .join + end + + def escape_ascii_shell_value(value) value.to_s.dump .gsub(/`/, '\\\\`') .gsub(DOLLAR_SIGN_WITHOUT_SHELL_EXPANSION_REGEX, '\$') diff --git a/test/cli/accessory_test.rb b/test/cli/accessory_test.rb index e56eef2d..0e3abc46 100644 --- a/test/cli/accessory_test.rb +++ b/test/cli/accessory_test.rb @@ -1,13 +1,21 @@ require_relative "cli_test_case" class CliAccessoryTest < CliTestCase + setup do + setup_test_secrets("secrets" => "MYSQL_ROOT_PASSWORD=secret") + end + + teardown do + teardown_test_secrets + end + test "boot" do Kamal::Cli::Accessory.any_instance.expects(:directories).with("mysql") Kamal::Cli::Accessory.any_instance.expects(:upload).with("mysql") run_command("boot", "mysql").tap do |output| assert_match /docker login.*on 1.1.1.3/, output - assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/env/accessories/app-mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output end end @@ -21,7 +29,7 @@ class CliAccessoryTest < CliTestCase assert_match /docker login.*on 1.1.1.3/, output assert_match /docker login.*on 1.1.1.1/, output assert_match /docker login.*on 1.1.1.2/, output - assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/env/accessories/app-mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output assert_match "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 8218dba8..85a966fd 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -85,7 +85,7 @@ class CliAppTest < CliTestCase run_command("boot", config: :with_assets).tap do |output| assert_match "docker tag dhh/app:latest dhh/app:latest", output assert_match "/usr/bin/env mkdir -p .kamal/assets/volumes/app-web-latest ; cp -rnT .kamal/assets/extracted/app-web-latest .kamal/assets/volumes/app-web-latest ; cp -rnT .kamal/assets/extracted/app-web-latest .kamal/assets/volumes/app-web-123 || true ; cp -rnT .kamal/assets/extracted/app-web-123 .kamal/assets/volumes/app-web-latest || true", output - assert_match "/usr/bin/env mkdir -p .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets 2> /dev/null || true && docker run --name app-web-assets --detach --rm dhh/app:latest sleep 1000000 && docker cp -L app-web-assets:/public/assets/. .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets", output + assert_match "/usr/bin/env mkdir -p .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets 2> /dev/null || true && docker run --name app-web-assets --detach --rm --entrypoint sleep dhh/app:latest 1000000 && docker cp -L app-web-assets:/public/assets/. .kamal/assets/extracted/app-web-latest && docker stop -t 1 app-web-assets", output assert_match /docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} /, output assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output assert_match "/usr/bin/env find .kamal/assets/extracted -maxdepth 1 -name 'app-web-*' ! -name app-web-latest -exec rm -rf \"{}\" + ; find .kamal/assets/volumes -maxdepth 1 -name 'app-web-*' ! -name app-web-latest -exec rm -rf \"{}\" +", output @@ -113,7 +113,7 @@ class CliAppTest < CliTestCase run_command("boot", config: :with_env_tags).tap do |output| assert_match "docker tag dhh/app:latest dhh/app:latest", output - assert_match %r{docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} -e KAMAL_CONTAINER_NAME="app-web-latest" -e KAMAL_VERSION="latest" --env-file .kamal/env/roles/app-web.env --env TEST="root" --env EXPERIMENT="disabled" --env SITE="site1"}, output + assert_match %r{docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} -e KAMAL_CONTAINER_NAME="app-web-latest" -e KAMAL_VERSION="latest" --env TEST="root" --env EXPERIMENT="disabled" --env SITE="site1"}, output assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output end end diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb index 4259fa5b..2d5d1051 100644 --- a/test/cli/build_test.rb +++ b/test/cli/build_test.rb @@ -49,7 +49,7 @@ class CliBuildTest < CliTestCase SSHKit::Backend::Abstract.any_instance.expects(:execute).with(:git, "-C", build_directory, :submodule, :update, "--init") SSHKit::Backend::Abstract.any_instance.expects(:execute) - .with(:docker, :buildx, :build, "--push", "--platform", "linux/amd64", "--builder", "kamal-local-docker-container", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".") + .with(:docker, :buildx, :build, "--push", "--platform", "linux/amd64", "--builder", "kamal-local-docker-container", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".", env: {}) SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:git, "-C", anything, :"rev-parse", :HEAD) @@ -140,7 +140,7 @@ class CliBuildTest < CliTestCase .returns("") SSHKit::Backend::Abstract.any_instance.expects(:execute) - .with(:docker, :buildx, :build, "--push", "--platform", "linux/amd64", "--builder", "kamal-local-docker-container", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".") + .with(:docker, :buildx, :build, "--push", "--platform", "linux/amd64", "--builder", "kamal-local-docker-container", "-t", "dhh/app:999", "-t", "dhh/app:latest", "--label", "service=\"app\"", "--file", "Dockerfile", ".", env: {}) run_command("push").tap do |output| assert_match /WARN Missing compatible builder, so creating a new one first/, output diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb index c522a32f..3f3e9294 100644 --- a/test/cli/cli_test_case.rb +++ b/test/cli/cli_test_case.rb @@ -40,7 +40,7 @@ class CliTestCase < ActiveSupport::TestCase .with(:docker, :buildx, :inspect, "kamal-local-docker-container") end - def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: false) + def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: false, secrets: false) whoami = `whoami`.chomp performer = Kamal::Git.email.presence || whoami service = service_version.split("@").first @@ -58,6 +58,7 @@ class CliTestCase < ActiveSupport::TestCase KAMAL_COMMAND=\"#{command}\"\s #{"KAMAL_SUBCOMMAND=\\\"#{subcommand}\\\"\\s" if subcommand} #{"KAMAL_RUNTIME=\\\"\\d+\\\"\\s" if runtime} + #{"DB_PASSWORD=\"secret\"\\s" if secrets} ;\s/usr/bin/env\s\.kamal/hooks/#{hook} }x assert_match expected, output diff --git a/test/cli/env_test.rb b/test/cli/env_test.rb deleted file mode 100644 index 299b2aaf..00000000 --- a/test/cli/env_test.rb +++ /dev/null @@ -1,37 +0,0 @@ -require_relative "cli_test_case" - -class CliEnvTest < CliTestCase - test "push" do - run_command("push").tap do |output| - assert_match "Running /usr/bin/env mkdir -p .kamal/env/roles on 1.1.1.1", output - assert_match "Running /usr/bin/env mkdir -p .kamal/env/traefik on 1.1.1.1", output - assert_match "Running /usr/bin/env mkdir -p .kamal/env/accessories on 1.1.1.1", output - assert_match "Running /usr/bin/env mkdir -p .kamal/env/roles on 1.1.1.1", output - assert_match "Running /usr/bin/env mkdir -p .kamal/env/traefik on 1.1.1.2", output - assert_match "Running /usr/bin/env mkdir -p .kamal/env/accessories on 1.1.1.1", output - assert_match ".kamal/env/roles/app-web.env", output - assert_match ".kamal/env/roles/app-workers.env", output - assert_match ".kamal/env/traefik/traefik.env", output - assert_match ".kamal/env/accessories/app-redis.env", output - end - end - - test "delete" do - run_command("delete").tap do |output| - assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-web.env on 1.1.1.1", output - assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-web.env on 1.1.1.2", output - assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-workers.env on 1.1.1.3", output - assert_match "Running /usr/bin/env rm -f .kamal/env/roles/app-workers.env on 1.1.1.4", output - assert_match "Running /usr/bin/env rm -f .kamal/env/traefik/traefik.env on 1.1.1.1", output - assert_match "Running /usr/bin/env rm -f .kamal/env/traefik/traefik.env on 1.1.1.2", output - assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-redis.env on 1.1.1.1", output - assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-redis.env on 1.1.1.2", output - assert_match "Running /usr/bin/env rm -f .kamal/env/accessories/app-mysql.env on 1.1.1.3", output - end - end - - private - def run_command(*command) - stdouted { Kamal::Cli::Env.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) } - end -end diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index 2b87191c..2115f418 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -8,14 +8,11 @@ class CliMainTest < CliTestCase invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options) - Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:main:envify", [], invoke_options) - Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options) Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options) Kamal::Cli::Main.any_instance.expects(:deploy) run_command("setup").tap do |output| assert_match /Ensure Docker is installed.../, output - assert_match /Evaluate and push env files.../, output end end @@ -23,8 +20,6 @@ class CliMainTest < CliTestCase invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options) - Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options) - Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:main:envify", [], invoke_options) Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options) # deploy Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: true)) @@ -36,7 +31,6 @@ class CliMainTest < CliTestCase run_command("setup", "--skip_push").tap do |output| assert_match /Ensure Docker is installed.../, output - assert_match /Evaluate and push env files.../, output # deploy assert_match /Acquiring the deploy lock/, output assert_match /Log into image registry/, output @@ -49,27 +43,29 @@ class CliMainTest < CliTestCase end test "deploy" do - invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true } + with_test_secrets("secrets" => "DB_PASSWORD=secret") do + invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false, "verbose" => true } - Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false)) - Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options) - Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options) - Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)) - Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options) - Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options.merge(skip_local: false)) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options) - Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) - hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "deploy" } + Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) + hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "deploy" } - run_command("deploy", "--verbose").tap do |output| - assert_hook_ran "pre-connect", output, **hook_variables - assert_match /Log into image registry/, output - assert_match /Build and push app image/, output - assert_hook_ran "pre-deploy", output, **hook_variables - assert_match /Ensure Traefik is running/, output - assert_match /Detect stale containers/, output - assert_match /Prune old containers and images/, output - assert_hook_ran "post-deploy", output, **hook_variables, runtime: true + run_command("deploy", "--verbose").tap do |output| + assert_hook_ran "pre-connect", output, **hook_variables + assert_match /Log into image registry/, output + assert_match /Build and push app image/, output + assert_hook_ran "pre-deploy", output, **hook_variables, secrets: true + assert_match /Ensure Traefik is running/, output + assert_match /Detect stale containers/, output + assert_match /Prune old containers and images/, output + assert_hook_ran "post-deploy", output, **hook_variables, runtime: true, secrets: true + end end end @@ -388,40 +384,38 @@ class CliMainTest < CliTestCase end test "init" do - Pathname.any_instance.expects(:exist?).returns(false).times(3) - Pathname.any_instance.stubs(:mkpath) - FileUtils.stubs(:mkdir_p) - FileUtils.stubs(:cp_r) - FileUtils.stubs(:cp) + in_dummy_git_repo do + run_command("init").tap do |output| + assert_match "Created configuration file in config/deploy.yml", output + assert_match "Created .kamal/secrets file", output + end - run_command("init").tap do |output| - assert_match /Created configuration file in config\/deploy.yml/, output - assert_match /Created \.env file/, output + assert_file "config/deploy.yml", "service: my-app" + assert_file ".kamal/secrets", "KAMAL_REGISTRY_PASSWORD=$KAMAL_REGISTRY_PASSWORD" end end test "init with existing config" do - Pathname.any_instance.expects(:exist?).returns(true).times(3) + in_dummy_git_repo do + run_command("init") - run_command("init").tap do |output| - assert_match /Config file already exists in config\/deploy.yml \(remove first to create a new one\)/, output + run_command("init").tap do |output| + assert_match /Config file already exists in config\/deploy.yml \(remove first to create a new one\)/, output + assert_no_match /Added .kamal\/secrets/, output + end end end test "init with bundle option" do - Pathname.any_instance.expects(:exist?).returns(false).times(4) - Pathname.any_instance.stubs(:mkpath) - FileUtils.stubs(:mkdir_p) - FileUtils.stubs(:cp_r) - FileUtils.stubs(:cp) - - run_command("init", "--bundle").tap do |output| - assert_match /Created configuration file in config\/deploy.yml/, output - assert_match /Created \.env file/, output - assert_match /Adding Kamal to Gemfile and bundle/, output - assert_match /bundle add kamal/, output - assert_match /bundle binstubs kamal/, output - assert_match /Created binstub file in bin\/kamal/, output + in_dummy_git_repo do + run_command("init", "--bundle").tap do |output| + assert_match "Created configuration file in config/deploy.yml", output + assert_match "Created .kamal/secrets file", output + assert_match /Adding Kamal to Gemfile and bundle/, output + assert_match /bundle add kamal/, output + assert_match /bundle binstubs kamal/, output + assert_match /Created binstub file in bin\/kamal/, output + end end end @@ -438,50 +432,6 @@ class CliMainTest < CliTestCase end end - test "envify" do - with_test_dotenv(".env.erb": "HELLO=<%= 'world' %>") do - run_command("envify") - assert_equal("HELLO=world", File.read(".env")) - end - end - - test "envify with blank line trimming" do - file = <<~EOF - HELLO=<%= 'world' %> - <% if true -%> - KEY=value - <% end -%> - EOF - - with_test_dotenv(".env.erb": file) do - run_command("envify") - assert_equal("HELLO=world\nKEY=value\n", File.read(".env")) - end - end - - test "envify with destination" do - with_test_dotenv(".env.world.erb": "HELLO=<%= 'world' %>") do - run_command("envify", "-d", "world", config_file: "deploy_for_dest") - assert_equal "HELLO=world", File.read(".env.world") - end - end - - test "envify with skip_push" do - Pathname.any_instance.expects(:exist?).returns(true).times(1) - File.expects(:read).with(".env.erb").returns("HELLO=<%= 'world' %>") - File.expects(:write).with(".env", "HELLO=world", perm: 0600) - - Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push").never - run_command("envify", "--skip-push") - end - - test "envify with clean env" do - with_test_dotenv(".env": "HELLO=already", ".env.erb": "HELLO=<%= ENV.fetch 'HELLO', 'never' %>") do - run_command("envify", "--skip-push") - assert_equal "HELLO=never", File.read(".env") - end - end - test "remove with confirmation" do run_command("remove", "-y", config_file: "deploy_with_accessories").tap do |output| assert_match /docker container stop traefik/, output @@ -572,18 +522,16 @@ class CliMainTest < CliTestCase end end - def with_test_dotenv(**files) - Dir.mktmpdir do |dir| - fixtures_dup = File.join(dir, "test") - FileUtils.mkdir_p(fixtures_dup) - FileUtils.cp_r("test/fixtures/", fixtures_dup) - - Dir.chdir(dir) do - files.each do |filename, contents| - File.binwrite(filename.to_s, contents) - end + def in_dummy_git_repo + Dir.mktmpdir do |tmpdir| + Dir.chdir(tmpdir) do + `git init` yield end end end + + def assert_file(file, content) + assert_match content, File.read(file) + end end diff --git a/test/cli/secrets_test.rb b/test/cli/secrets_test.rb new file mode 100644 index 00000000..35d20500 --- /dev/null +++ b/test/cli/secrets_test.rb @@ -0,0 +1,22 @@ +require_relative "cli_test_case" + +class CliSecretsTest < CliTestCase + test "fetch" do + assert_equal \ + "\\{\\\"foo\\\":\\\"oof\\\",\\\"bar\\\":\\\"rab\\\",\\\"baz\\\":\\\"zab\\\"\\}", + run_command("fetch", "foo", "bar", "baz", "--account", "myaccount", "--adapter", "test") + end + + test "extract" do + assert_equal "oof", run_command("extract", "foo", "{\"foo\":\"oof\", \"bar\":\"rab\", \"baz\":\"zab\"}") + end + + test "extract match from end" do + assert_equal "oof", run_command("extract", "foo", "{\"abc/foo\":\"oof\", \"bar\":\"rab\", \"baz\":\"zab\"}") + end + + private + def run_command(*command) + stdouted { Kamal::Cli::Secrets.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) } + end +end diff --git a/test/commands/accessory_test.rb b/test/commands/accessory_test.rb index 40002988..23d304da 100644 --- a/test/commands/accessory_test.rb +++ b/test/commands/accessory_test.rb @@ -2,6 +2,8 @@ require "test_helper" class CommandsAccessoryTest < ActiveSupport::TestCase setup do + setup_test_secrets("secrets" => "MYSQL_ROOT_PASSWORD=secret123") + @config = { service: "app", image: "dhh/app", registry: { "server" => "private.registry", "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], @@ -41,21 +43,19 @@ class CommandsAccessoryTest < ActiveSupport::TestCase } } } - - ENV["MYSQL_ROOT_PASSWORD"] = "secret123" end teardown do - ENV.delete("MYSQL_ROOT_PASSWORD") + teardown_test_secrets end test "run" do assert_equal \ - "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" --label service=\"app-mysql\" private.registry/mysql:8.0", + "docker run --name app-mysql --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/env/accessories/app-mysql.env --label service=\"app-mysql\" private.registry/mysql:8.0", new_command(:mysql).run.join(" ") assert_equal \ - "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/env/accessories/app-redis.env --env SOMETHING=\"else\" --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest", + "docker run --name app-redis --detach --restart unless-stopped --log-opt max-size=\"10m\" --publish 6379:6379 --env SOMETHING=\"else\" --env-file .kamal/env/accessories/app-redis.env --volume /var/lib/redis:/data --label service=\"app-redis\" --label cache=\"true\" redis:latest", new_command(:redis).run.join(" ") assert_equal \ @@ -92,7 +92,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase test "execute in new container" do assert_equal \ - "docker run --rm --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" private.registry/mysql:8.0 mysql -u root", + "docker run --rm --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/env/accessories/app-mysql.env private.registry/mysql:8.0 mysql -u root", new_command(:mysql).execute_in_new_container("mysql", "-u", "root").join(" ") end @@ -104,7 +104,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase test "execute in new container over ssh" do new_command(:mysql).stub(:run_over_ssh, ->(cmd) { cmd.join(" ") }) do - assert_match %r{docker run -it --rm --env-file .kamal/env/accessories/app-mysql.env --env MYSQL_ROOT_HOST=\"%\" private.registry/mysql:8.0 mysql -u root}, + assert_match %r{docker run -it --rm --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/env/accessories/app-mysql.env private.registry/mysql:8.0 mysql -u root}, new_command(:mysql).execute_in_new_container_over_ssh("mysql", "-u", "root") end end @@ -150,14 +150,6 @@ class CommandsAccessoryTest < ActiveSupport::TestCase new_command(:mysql).remove_image.join(" ") end - test "make_env_directory" do - assert_equal "mkdir -p .kamal/env/accessories", new_command(:mysql).make_env_directory.join(" ") - end - - test "remove_env_file" do - assert_equal "rm -f .kamal/env/accessories/app-mysql.env", new_command(:mysql).remove_env_file.join(" ") - end - private def new_command(accessory) Kamal::Commands::Accessory.new(Kamal::Configuration.new(@config), name: accessory) diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index bee7dc34..e385764e 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -2,14 +2,14 @@ require "test_helper" class CommandsAppTest < ActiveSupport::TestCase setup do - ENV["RAILS_MASTER_KEY"] = "456" + setup_test_secrets("secrets" => "RAILS_MASTER_KEY=456") Kamal::Configuration.any_instance.stubs(:run_id).returns("12345678901234567890123456789012") @config = { service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], env: { "secret" => [ "RAILS_MASTER_KEY" ] }, builder: { "arch" => "amd64" } } end teardown do - ENV.delete("RAILS_MASTER_KEY") + teardown_test_secrets end test "run" do @@ -85,7 +85,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:env]["tags"] = { "tag1" => { "ENV1" => "value1" } } assert_equal \ - "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", + "docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env ENV1=\"value1\" --env-file .kamal/env/roles/app-web.env --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999", new_command.run.join(" ") end @@ -219,7 +219,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:env]["tags"] = { "tag1" => { "ENV1" => "value1" } } assert_equal \ - "docker run --rm --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" dhh/app:999 bin/rails db:setup", + "docker run --rm --env ENV1=\"value1\" --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails db:setup", new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ") end @@ -251,7 +251,7 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:servers] = [ { "1.1.1.1" => "tag1" } ] @config[:env]["tags"] = { "tag1" => { "ENV1" => "value1" } } - assert_equal "ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" dhh/app:999 bin/rails c'", + assert_equal "ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --env ENV1=\"value1\" --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails c'", new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {}) end @@ -412,14 +412,6 @@ class CommandsAppTest < ActiveSupport::TestCase new_command.tag_latest_image.join(" ") end - test "make_env_directory" do - assert_equal "mkdir -p .kamal/env/roles", new_command.make_env_directory.join(" ") - end - - test "remove_env_file" do - assert_equal "rm -f .kamal/env/roles/app-web.env", new_command.remove_env_file.join(" ") - end - test "cord" do assert_equal "docker inspect -f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}' app-web-123 | awk '$2 == \"/tmp/kamal-cord\" {print $1}'", new_command.cord(version: 123).join(" ") end @@ -438,7 +430,7 @@ class CommandsAppTest < ActiveSupport::TestCase assert_equal [ :mkdir, "-p", ".kamal/assets/extracted/app-web-999", "&&", :docker, :stop, "-t 1", "app-web-assets", "2> /dev/null", "|| true", "&&", - :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "dhh/app:999", "sleep 1000000", "&&", + :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "--entrypoint", "sleep", "dhh/app:999", "1000000", "&&", :docker, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/assets/extracted/app-web-999", "&&", :docker, :stop, "-t 1", "app-web-assets" ], new_command(asset_path: "/public/assets").extract_assets diff --git a/test/commands/auditor_test.rb b/test/commands/auditor_test.rb index 2aaafd67..2abc8d81 100644 --- a/test/commands/auditor_test.rb +++ b/test/commands/auditor_test.rb @@ -18,6 +18,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase test "record" do assert_equal [ + :mkdir, "-p", ".kamal", "&&", :echo, "[#{@recorded_at}] [#{@performer}]", "app removed container", @@ -28,6 +29,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase test "record with destination" do new_command(destination: "staging").tap do |auditor| assert_equal [ + :mkdir, "-p", ".kamal", "&&", :echo, "[#{@recorded_at}] [#{@performer}] [staging]", "app removed container", @@ -39,6 +41,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase test "record with command details" do new_command(role: "web").tap do |auditor| assert_equal [ + :mkdir, "-p", ".kamal", "&&", :echo, "[#{@recorded_at}] [#{@performer}] [web]", "app removed container", @@ -49,6 +52,7 @@ class CommandsAuditorTest < ActiveSupport::TestCase test "record with arg details" do assert_equal [ + :mkdir, "-p", ".kamal", "&&", :echo, "[#{@recorded_at}] [#{@performer}] [value]", "app removed container", diff --git a/test/commands/builder_test.rb b/test/commands/builder_test.rb index aff8c0c6..7102c672 100644 --- a/test/commands/builder_test.rb +++ b/test/commands/builder_test.rb @@ -77,10 +77,13 @@ class CommandsBuilderTest < ActiveSupport::TestCase end test "build secrets" do - builder = new_builder_command(builder: { "secrets" => [ "token_a", "token_b" ] }) - assert_equal \ - "-t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"token_a\" --secret id=\"token_b\" --file Dockerfile", - builder.target.build_options.join(" ") + with_test_secrets("secrets" => "token_a=foo\ntoken_b=bar") do + FileUtils.touch("Dockerfile") + builder = new_builder_command(builder: { "secrets" => [ "token_a", "token_b" ] }) + assert_equal \ + "-t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"token_a\" --secret id=\"token_b\" --file Dockerfile", + builder.target.build_options.join(" ") + end end test "build dockerfile" do @@ -121,10 +124,13 @@ class CommandsBuilderTest < ActiveSupport::TestCase end test "push with build secrets" do - builder = new_builder_command(builder: { "secrets" => [ "a", "b" ] }) - assert_equal \ - "docker buildx build --push --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile .", - builder.push.join(" ") + with_test_secrets("secrets" => "a=foo\nb=bar") do + FileUtils.touch("Dockerfile") + builder = new_builder_command(builder: { "secrets" => [ "a", "b" ] }) + assert_equal \ + "docker buildx build --push --platform linux/amd64 --builder kamal-local-docker-container -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --secret id=\"a\" --secret id=\"b\" --file Dockerfile .", + builder.push.join(" ") + end end test "build with ssh agent socket" do diff --git a/test/commands/hook_test.rb b/test/commands/hook_test.rb index 60438c66..f6234d6a 100644 --- a/test/commands/hook_test.rb +++ b/test/commands/hook_test.rb @@ -39,6 +39,21 @@ class CommandsHookTest < ActiveSupport::TestCase ], new_command(hooks_path: "custom/hooks/path").run("foo") end + test "hook with secrets" do + with_test_secrets("secrets" => "DB_PASSWORD=secret") do + assert_equal [ + ".kamal/hooks/foo", + { env: { + "KAMAL_RECORDED_AT" => @recorded_at, + "KAMAL_PERFORMER" => @performer, + "KAMAL_VERSION" => "123", + "KAMAL_SERVICE_VERSION" => "app@123", + "KAMAL_SERVICE" => "app", + "DB_PASSWORD" => "secret" } } + ], new_command(env: { "secret" => [ "DB_PASSWORD" ] }).run("foo", secrets: true) + end + end + private def new_command(**extra_config) Kamal::Commands::Hook.new(Kamal::Configuration.new(@config.merge(**extra_config), version: "123")) diff --git a/test/commands/registry_test.rb b/test/commands/registry_test.rb index 17376fef..cf2734b7 100755 --- a/test/commands/registry_test.rb +++ b/test/commands/registry_test.rb @@ -11,51 +11,52 @@ class CommandsRegistryTest < ActiveSupport::TestCase builder: { "arch" => "amd64" }, servers: [ "1.1.1.1" ] } - @registry = Kamal::Commands::Registry.new Kamal::Configuration.new(@config) end test "registry login" do assert_equal \ "docker login hub.docker.com -u \"dhh\" -p \"secret\"", - @registry.login.join(" ") + registry.login.join(" ") end test "registry login with ENV password" do - ENV["KAMAL_REGISTRY_PASSWORD"] = "more-secret" - @config[:registry]["password"] = [ "KAMAL_REGISTRY_PASSWORD" ] + with_test_secrets("secrets" => "KAMAL_REGISTRY_PASSWORD=more-secret") do + @config[:registry]["password"] = [ "KAMAL_REGISTRY_PASSWORD" ] - assert_equal \ - "docker login hub.docker.com -u \"dhh\" -p \"more-secret\"", - @registry.login.join(" ") - ensure - ENV.delete("KAMAL_REGISTRY_PASSWORD") + assert_equal \ + "docker login hub.docker.com -u \"dhh\" -p \"more-secret\"", + registry.login.join(" ") + end end test "registry login escape password" do - ENV["KAMAL_REGISTRY_PASSWORD"] = "more-secret'\"" - @config[:registry]["password"] = [ "KAMAL_REGISTRY_PASSWORD" ] + with_test_secrets("secrets" => "KAMAL_REGISTRY_PASSWORD=more-secret'\"") do + @config[:registry]["password"] = [ "KAMAL_REGISTRY_PASSWORD" ] - assert_equal \ - "docker login hub.docker.com -u \"dhh\" -p \"more-secret'\\\"\"", - @registry.login.join(" ") - ensure - ENV.delete("KAMAL_REGISTRY_PASSWORD") + assert_equal \ + "docker login hub.docker.com -u \"dhh\" -p \"more-secret'\\\"\"", + registry.login.join(" ") + end end test "registry login with ENV username" do - ENV["KAMAL_REGISTRY_USERNAME"] = "also-secret" - @config[:registry]["username"] = [ "KAMAL_REGISTRY_USERNAME" ] + with_test_secrets("secrets" => "KAMAL_REGISTRY_USERNAME=also-secret") do + @config[:registry]["username"] = [ "KAMAL_REGISTRY_USERNAME" ] - assert_equal \ - "docker login hub.docker.com -u \"also-secret\" -p \"secret\"", - @registry.login.join(" ") - ensure - ENV.delete("KAMAL_REGISTRY_USERNAME") + assert_equal \ + "docker login hub.docker.com -u \"also-secret\" -p \"secret\"", + registry.login.join(" ") + end end test "registry logout" do assert_equal \ "docker logout hub.docker.com", - @registry.logout.join(" ") + registry.logout.join(" ") end + + private + def registry + Kamal::Commands::Registry.new Kamal::Configuration.new(@config) + end end diff --git a/test/commands/traefik_test.rb b/test/commands/traefik_test.rb index 446c3077..b13e3700 100644 --- a/test/commands/traefik_test.rb +++ b/test/commands/traefik_test.rb @@ -9,11 +9,11 @@ class CommandsTraefikTest < ActiveSupport::TestCase traefik: { "image" => @image, "args" => { "accesslog.format" => "json", "api.insecure" => true, "metrics.prometheus.buckets" => "0.1,0.3,1.2,5.0" } } } - ENV["EXAMPLE_API_KEY"] = "456" + setup_test_secrets("secrets" => "EXAMPLE_API_KEY=456") end teardown do - ENV.delete("EXAMPLE_API_KEY") + teardown_test_secrets end test "run" do @@ -81,9 +81,9 @@ class CommandsTraefikTest < ActiveSupport::TestCase "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") - @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] } + @config[:traefik]["env"] = { "EXAMPLE_API_KEY" => "456" } assert_equal \ - "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", + "docker run --name traefik --detach --restart unless-stopped --publish 80:80 --volume /var/run/docker.sock:/var/run/docker.sock --env EXAMPLE_API_KEY=\"456\" --env-file .kamal/env/traefik/traefik.env --log-opt max-size=\"10m\" --label traefik.http.routers.catchall.entryPoints=\"http\" --label traefik.http.routers.catchall.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.catchall.service=\"unavailable\" --label traefik.http.routers.catchall.priority=\"1\" --label traefik.http.services.unavailable.loadbalancer.server.port=\"0\" #{@image} --providers.docker --log.level=\"DEBUG\" --accesslog.format=\"json\" --api.insecure --metrics.prometheus.buckets=\"0.1,0.3,1.2,5.0\"", new_command.run.join(" ") end @@ -188,20 +188,6 @@ class CommandsTraefikTest < ActiveSupport::TestCase new_command.follow_logs(host: @config[:servers].first, grep: "hello!") end - test "secrets io" do - @config[:traefik]["env"] = { "secret" => %w[EXAMPLE_API_KEY] } - - assert_equal "EXAMPLE_API_KEY=456\n", new_command.env.secrets_io.string - end - - test "make_env_directory" do - assert_equal "mkdir -p .kamal/env/traefik", new_command.make_env_directory.join(" ") - end - - test "remove_env_file" do - assert_equal "rm -f .kamal/env/traefik/traefik.env", new_command.remove_env_file.join(" ") - end - private def new_command Kamal::Commands::Traefik.new(Kamal::Configuration.new(@config, version: "123")) diff --git a/test/configuration/accessory_test.rb b/test/configuration/accessory_test.rb index 3f939607..acfe991f 100644 --- a/test/configuration/accessory_test.rb +++ b/test/configuration/accessory_test.rb @@ -116,25 +116,14 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase end test "env args" do - assert_equal [ "--env-file", ".kamal/env/accessories/app-mysql.env", "--env", "MYSQL_ROOT_HOST=\"%\"" ], @config.accessory(:mysql).env_args - assert_equal [ "--env-file", ".kamal/env/accessories/app-redis.env", "--env", "SOMETHING=\"else\"" ], @config.accessory(:redis).env_args - end + with_test_secrets("secrets" => "MYSQL_ROOT_PASSWORD=secret123") do + config = Kamal::Configuration.new(@deploy) - test "env with secrets" do - ENV["MYSQL_ROOT_PASSWORD"] = "secret123" - - expected_secrets_file = <<~ENV - MYSQL_ROOT_PASSWORD=secret123 - ENV - - assert_equal expected_secrets_file, @config.accessory(:mysql).env.secrets_io.string - assert_equal [ "--env-file", ".kamal/env/accessories/app-mysql.env", "--env", "MYSQL_ROOT_HOST=\"%\"" ], @config.accessory(:mysql).env_args - ensure - ENV["MYSQL_ROOT_PASSWORD"] = nil - end - - test "env secrets path" do - assert_equal ".kamal/env/accessories/app-mysql.env", @config.accessory(:mysql).env.secrets_file + assert_equal [ "--env", "MYSQL_ROOT_HOST=\"%\"", "--env-file", ".kamal/env/accessories/app-mysql.env" ], config.accessory(:mysql).env_args.map(&:to_s) + assert_equal "MYSQL_ROOT_PASSWORD=secret123\n", config.accessory(:mysql).secrets_io.string + assert_equal [ "--env", "SOMETHING=\"else\"", "--env-file", ".kamal/env/accessories/app-redis.env" ], @config.accessory(:redis).env_args + assert_equal "\n", config.accessory(:redis).secrets_io.string + end end test "volume args" do diff --git a/test/configuration/builder_test.rb b/test/configuration/builder_test.rb index 0aab0216..a6a754f0 100644 --- a/test/configuration/builder_test.rb +++ b/test/configuration/builder_test.rb @@ -110,13 +110,15 @@ class ConfigurationBuilderTest < ActiveSupport::TestCase end test "secrets" do - assert_equal [], config.builder.secrets + assert_equal({}, config.builder.secrets) end test "setting secrets" do - @deploy[:builder]["secrets"] = [ "GITHUB_TOKEN" ] + with_test_secrets("secrets" => "GITHUB_TOKEN=secret123") do + @deploy[:builder]["secrets"] = [ "GITHUB_TOKEN" ] - assert_equal [ "GITHUB_TOKEN" ], config.builder.secrets + assert_equal({ "GITHUB_TOKEN" => "secret123" }, config.builder.secrets) + end end test "dockerfile" do diff --git a/test/configuration/env/tags_test.rb b/test/configuration/env/tags_test.rb index 0fb649d1..7db61777 100644 --- a/test/configuration/env/tags_test.rb +++ b/test/configuration/env/tags_test.rb @@ -79,23 +79,21 @@ class ConfigurationEnvTagsTest < ActiveSupport::TestCase end test "tag secret env" do - ENV["PASSWORD"] = "hello" - - deploy = { - service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, - servers: [ { "1.1.1.1" => "secrets" } ], - builder: { "arch" => "amd64" }, - env: { - "tags" => { - "secrets" => { "secret" => [ "PASSWORD" ] } + with_test_secrets("secrets" => "PASSWORD=hello") do + deploy = { + service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, + servers: [ { "1.1.1.1" => "secrets" } ], + builder: { "arch" => "amd64" }, + env: { + "tags" => { + "secrets" => { "secret" => [ "PASSWORD" ] } + } } } - } - config = Kamal::Configuration.new(deploy) - assert_equal "hello", config.role("web").env("1.1.1.1").secrets["PASSWORD"] - ensure - ENV.delete "PASSWORD" + config = Kamal::Configuration.new(deploy) + assert_equal "hello", config.role("web").env("1.1.1.1").secrets["PASSWORD"] + end end test "tag clear env" do diff --git a/test/configuration/env_test.rb b/test/configuration/env_test.rb index 49d800ef..627d3a6c 100644 --- a/test/configuration/env_test.rb +++ b/test/configuration/env_test.rb @@ -6,27 +6,21 @@ class ConfigurationEnvTest < ActiveSupport::TestCase test "simple" do assert_config \ config: { "foo" => "bar", "baz" => "haz" }, - clear: { "foo" => "bar", "baz" => "haz" }, - secrets: {} + clear: { "foo" => "bar", "baz" => "haz" } end test "clear" do assert_config \ config: { "clear" => { "foo" => "bar", "baz" => "haz" } }, - clear: { "foo" => "bar", "baz" => "haz" }, - secrets: {} + clear: { "foo" => "bar", "baz" => "haz" } end test "secret" do - ENV["PASSWORD"] = "hello" - env = Kamal::Configuration::Env.new config: { "secret" => [ "PASSWORD" ] } - - assert_config \ - config: { "secret" => [ "PASSWORD" ] }, - clear: {}, - secrets: { "PASSWORD" => "hello" } - ensure - ENV.delete "PASSWORD" + with_test_secrets("secrets" => "PASSWORD=hello") do + assert_config \ + config: { "secret" => [ "PASSWORD" ] }, + secrets: { "PASSWORD" => "hello" } + end end test "missing secret" do @@ -34,41 +28,32 @@ class ConfigurationEnvTest < ActiveSupport::TestCase "secret" => [ "PASSWORD" ] } - assert_raises(KeyError) { Kamal::Configuration::Env.new(config: { "secret" => [ "PASSWORD" ] }).secrets } + assert_raises(Kamal::ConfigurationError) { Kamal::Configuration::Env.new(config: { "secret" => [ "PASSWORD" ] }, secrets: Kamal::Secrets.new).secrets_io } end test "secret and clear" do - ENV["PASSWORD"] = "hello" - config = { - "secret" => [ "PASSWORD" ], - "clear" => { - "foo" => "bar", - "baz" => "haz" + with_test_secrets("secrets" => "PASSWORD=hello") do + config = { + "secret" => [ "PASSWORD" ], + "clear" => { + "foo" => "bar", + "baz" => "haz" + } } - } - assert_config \ - config: config, - clear: { "foo" => "bar", "baz" => "haz" }, - secrets: { "PASSWORD" => "hello" } - ensure - ENV.delete "PASSWORD" - end - - test "stringIO conversion" do - env = { - "foo" => "bar", - "baz" => "haz" - } - - assert_equal "foo=bar\nbaz=haz\n", \ - StringIO.new(Kamal::EnvFile.new(env)).read + assert_config \ + config: config, + clear: { "foo" => "bar", "baz" => "haz" }, + secrets: { "PASSWORD" => "hello" } + end end private - def assert_config(config:, clear:, secrets:) - env = Kamal::Configuration::Env.new config: config, secrets_file: "secrets.env" - assert_equal clear, env.clear - assert_equal secrets, env.secrets + def assert_config(config:, clear: {}, secrets: {}) + env = Kamal::Configuration::Env.new config: config, secrets: Kamal::Secrets.new + expected_clear_args = clear.to_a.flat_map { |key, value| [ "--env", "#{key}=\"#{value}\"" ] } + assert_equal expected_clear_args, env.clear_args.map(&:to_s) #  to_s removes the redactions + expected_secrets = secrets.to_a.flat_map { |key, value| "#{key}=#{value}" }.join("\n") + "\n" + assert_equal expected_secrets, env.secrets_io.string end end diff --git a/test/configuration/role_test.rb b/test/configuration/role_test.rb index 37c26fcd..c0b643bf 100644 --- a/test/configuration/role_test.rb +++ b/test/configuration/role_test.rb @@ -9,8 +9,6 @@ class ConfigurationRoleTest < ActiveSupport::TestCase env: { "REDIS_URL" => "redis://x/y" } } - @config = Kamal::Configuration.new(@deploy) - @deploy_with_roles = @deploy.dup.merge({ servers: { "web" => [ "1.1.1.1", "1.1.1.2" ], @@ -24,31 +22,29 @@ class ConfigurationRoleTest < ActiveSupport::TestCase } } }) - - @config_with_roles = Kamal::Configuration.new(@deploy_with_roles) end test "hosts" do - assert_equal [ "1.1.1.1", "1.1.1.2" ], @config.role(:web).hosts - assert_equal [ "1.1.1.3", "1.1.1.4" ], @config_with_roles.role(:workers).hosts + assert_equal [ "1.1.1.1", "1.1.1.2" ], config.role(:web).hosts + assert_equal [ "1.1.1.3", "1.1.1.4" ], config_with_roles.role(:workers).hosts end test "cmd" do - assert_nil @config.role(:web).cmd - assert_equal "bin/jobs", @config_with_roles.role(:workers).cmd + assert_nil config.role(:web).cmd + assert_equal "bin/jobs", config_with_roles.role(:workers).cmd end test "label args" do - assert_equal [ "--label", "service=\"app\"", "--label", "role=\"workers\"", "--label", "destination" ], @config_with_roles.role(:workers).label_args + assert_equal [ "--label", "service=\"app\"", "--label", "role=\"workers\"", "--label", "destination" ], config_with_roles.role(:workers).label_args end test "special label args for web" do - assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "destination", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.routers.app-web.priority=\"2\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], @config.role(:web).label_args + assert_equal [ "--label", "service=\"app\"", "--label", "role=\"web\"", "--label", "destination", "--label", "traefik.http.services.app-web.loadbalancer.server.scheme=\"http\"", "--label", "traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\"", "--label", "traefik.http.routers.app-web.priority=\"2\"", "--label", "traefik.http.middlewares.app-web-retry.retry.attempts=\"5\"", "--label", "traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\"", "--label", "traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\"" ], config.role(:web).label_args end test "custom labels" do @deploy[:labels] = { "my.custom.label" => "50" } - assert_equal "50", @config.role(:web).labels["my.custom.label"] + assert_equal "50", config.role(:web).labels["my.custom.label"] end test "custom labels via role specialization" do @@ -59,7 +55,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase test "overwriting default traefik label" do @deploy[:labels] = { "traefik.http.routers.app-web.rule" => "\"Host(\\`example.com\\`) || (Host(\\`example.org\\`) && Path(\\`/traefik\\`))\"" } - assert_equal "\"Host(\\`example.com\\`) || (Host(\\`example.org\\`) && Path(\\`/traefik\\`))\"", @config.role(:web).labels["traefik.http.routers.app-web.rule"] + assert_equal "\"Host(\\`example.com\\`) || (Host(\\`example.org\\`) && Path(\\`/traefik\\`))\"", config.role(:web).labels["traefik.http.routers.app-web.rule"] end test "default traefik label on non-web role" do @@ -71,166 +67,165 @@ class ConfigurationRoleTest < ActiveSupport::TestCase end test "env overwritten by role" do - assert_equal "redis://a/b", @config_with_roles.role(:workers).env("1.1.1.3").clear["REDIS_URL"] + assert_equal "redis://a/b", config_with_roles.role(:workers).env("1.1.1.3").clear["REDIS_URL"] - assert_equal "\n", @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string - assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3") + assert_equal \ + [ "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"", "--env-file", ".kamal/env/roles/app-workers.env" ], + config_with_roles.role(:workers).env_args("1.1.1.3").map(&:to_s) + + assert_equal \ + "\n", + config_with_roles.role(:workers).secrets_io("1.1.1.3").read end test "container name" do ENV["VERSION"] = "12345" - assert_equal "app-workers-12345", @config_with_roles.role(:workers).container_name - assert_equal "app-web-12345", @config_with_roles.role(:web).container_name + assert_equal "app-workers-12345", config_with_roles.role(:workers).container_name + assert_equal "app-web-12345", config_with_roles.role(:web).container_name ensure ENV.delete("VERSION") end test "env args" do - assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3") + assert_equal \ + [ "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"", "--env-file", ".kamal/env/roles/app-workers.env" ], + config_with_roles.role(:workers).env_args("1.1.1.3").map(&:to_s) + + assert_equal \ + "\n", + config_with_roles.role(:workers).secrets_io("1.1.1.3").read end test "env secret overwritten by role" do - @deploy_with_roles[:env] = { - "clear" => { - "REDIS_URL" => "redis://a/b" - }, - "secret" => [ - "REDIS_PASSWORD" - ] - } + with_test_secrets("secrets" => "REDIS_PASSWORD=secret456\nDB_PASSWORD=secret&\"123") do + @deploy_with_roles[:env] = { + "clear" => { + "REDIS_URL" => "redis://a/b" + }, + "secret" => [ + "REDIS_PASSWORD" + ] + } - @deploy_with_roles[:servers]["workers"]["env"] = { - "clear" => { - "REDIS_URL" => "redis://a/b", - "WEB_CONCURRENCY" => "4" - }, - "secret" => [ - "DB_PASSWORD" - ] - } + @deploy_with_roles[:servers]["workers"]["env"] = { + "clear" => { + "REDIS_URL" => "redis://a/b", + "WEB_CONCURRENCY" => "4" + }, + "secret" => [ + "DB_PASSWORD" + ] + } - ENV["REDIS_PASSWORD"] = "secret456" - ENV["DB_PASSWORD"] = "secret&\"123" + assert_equal \ + [ "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"", "--env-file", ".kamal/env/roles/app-workers.env" ], + config_with_roles.role(:workers).env_args("1.1.1.3").map(&:to_s) - expected_secrets_file = <<~ENV - REDIS_PASSWORD=secret456 - DB_PASSWORD=secret&\"123 - ENV - - assert_equal expected_secrets_file, Kamal::Configuration.new(@deploy_with_roles).role(:workers).env("1.1.1.3").secrets_io.string - assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3") - ensure - ENV["REDIS_PASSWORD"] = nil - ENV["DB_PASSWORD"] = nil + assert_equal \ + "REDIS_PASSWORD=secret456\nDB_PASSWORD=secret&\"123\n", + config_with_roles.role(:workers).secrets_io("1.1.1.3").read + end end test "env secrets only in role" do - @deploy_with_roles[:servers]["workers"]["env"] = { - "clear" => { - "REDIS_URL" => "redis://a/b", - "WEB_CONCURRENCY" => "4" - }, - "secret" => [ - "DB_PASSWORD" - ] - } + with_test_secrets("secrets" => "DB_PASSWORD=secret123") do + @deploy_with_roles[:servers]["workers"]["env"] = { + "clear" => { + "REDIS_URL" => "redis://a/b", + "WEB_CONCURRENCY" => "4" + }, + "secret" => [ + "DB_PASSWORD" + ] + } - ENV["DB_PASSWORD"] = "secret123" + assert_equal \ + [ "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"", "--env-file", ".kamal/env/roles/app-workers.env" ], + config_with_roles.role(:workers).env_args("1.1.1.3").map(&:to_s) - expected_secrets_file = <<~ENV - DB_PASSWORD=secret123 - ENV - - assert_equal expected_secrets_file, Kamal::Configuration.new(@deploy_with_roles).role(:workers).env("1.1.1.3").secrets_io.string - assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3") - ensure - ENV["DB_PASSWORD"] = nil + assert_equal \ + "DB_PASSWORD=secret123\n", + config_with_roles.role(:workers).secrets_io("1.1.1.3").read + end end test "env secrets only at top level" do - @deploy_with_roles[:env] = { - "clear" => { - "REDIS_URL" => "redis://a/b" - }, - "secret" => [ - "REDIS_PASSWORD" - ] - } + with_test_secrets("secrets" => "REDIS_PASSWORD=secret456") do + @deploy_with_roles[:env] = { + "clear" => { + "REDIS_URL" => "redis://a/b" + }, + "secret" => [ + "REDIS_PASSWORD" + ] + } - ENV["REDIS_PASSWORD"] = "secret456" + assert_equal \ + [ "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"", "--env-file", ".kamal/env/roles/app-workers.env" ], + config_with_roles.role(:workers).env_args("1.1.1.3").map(&:to_s) - expected_secrets_file = <<~ENV - REDIS_PASSWORD=secret456 - ENV - - assert_equal expected_secrets_file, Kamal::Configuration.new(@deploy_with_roles).role(:workers).env("1.1.1.3").secrets_io.string - assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3") - ensure - ENV["REDIS_PASSWORD"] = nil + assert_equal \ + "REDIS_PASSWORD=secret456\n", + config_with_roles.role(:workers).secrets_io("1.1.1.3").read + end end test "env overwritten by role with secrets" do - @deploy_with_roles[:env] = { - "clear" => { - "REDIS_URL" => "redis://a/b" - }, - "secret" => [ - "REDIS_PASSWORD" - ] - } - - @deploy_with_roles[:servers]["workers"]["env"] = { - "clear" => { - "REDIS_URL" => "redis://c/d" + with_test_secrets("secrets" => "REDIS_PASSWORD=secret456") do + @deploy_with_roles[:env] = { + "clear" => { + "REDIS_URL" => "redis://a/b" + }, + "secret" => [ + "REDIS_PASSWORD" + ] } - } - ENV["REDIS_PASSWORD"] = "secret456" + @deploy_with_roles[:servers]["workers"]["env"] = { + "clear" => { + "REDIS_URL" => "redis://c/d" + } + } - expected_secrets_file = <<~ENV - REDIS_PASSWORD=secret456 - ENV + assert_equal \ + [ "--env", "REDIS_URL=\"redis://c/d\"", "--env-file", ".kamal/env/roles/app-workers.env" ], + config_with_roles.role(:workers).env_args("1.1.1.3").map(&:to_s) - config = Kamal::Configuration.new(@deploy_with_roles) - assert_equal expected_secrets_file, config.role(:workers).env("1.1.1.3").secrets_io.string - assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://c/d\"" ], config.role(:workers).env_args("1.1.1.3") - ensure - ENV["REDIS_PASSWORD"] = nil - end - - test "env secrets_file" do - assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).env("1.1.1.3").secrets_file + assert_equal \ + "REDIS_PASSWORD=secret456\n", + config_with_roles.role(:workers).secrets_io("1.1.1.3").read + end end test "uses cord" do - assert @config_with_roles.role(:web).uses_cord? - assert_not @config_with_roles.role(:workers).uses_cord? + assert config_with_roles.role(:web).uses_cord? + assert_not config_with_roles.role(:workers).uses_cord? end test "cord host file" do - assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}/cord}, @config_with_roles.role(:web).cord_host_file + assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}/cord}, config_with_roles.role(:web).cord_host_file end test "cord volume" do - assert_equal "/tmp/kamal-cord", @config_with_roles.role(:web).cord_volume.container_path - assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}}, @config_with_roles.role(:web).cord_volume.host_path - assert_equal "--volume", @config_with_roles.role(:web).cord_volume.docker_args[0] - assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}:/tmp/kamal-cord}, @config_with_roles.role(:web).cord_volume.docker_args[1] + assert_equal "/tmp/kamal-cord", config_with_roles.role(:web).cord_volume.container_path + assert_match %r{.kamal/cords/app-web-[0-9a-f]{32}}, config_with_roles.role(:web).cord_volume.host_path + assert_equal "--volume", config_with_roles.role(:web).cord_volume.docker_args[0] + assert_match %r{\$\(pwd\)/.kamal/cords/app-web-[0-9a-f]{32}:/tmp/kamal-cord}, config_with_roles.role(:web).cord_volume.docker_args[1] end test "cord container file" do - assert_equal "/tmp/kamal-cord/cord", @config_with_roles.role(:web).cord_container_file + assert_equal "/tmp/kamal-cord/cord", config_with_roles.role(:web).cord_container_file end test "asset path and volume args" do ENV["VERSION"] = "12345" - assert_nil @config_with_roles.role(:web).asset_volume_args - assert_nil @config_with_roles.role(:workers).asset_volume_args - assert_nil @config_with_roles.role(:web).asset_path - assert_nil @config_with_roles.role(:workers).asset_path - assert_not @config_with_roles.role(:web).assets? - assert_not @config_with_roles.role(:workers).assets? + assert_nil config_with_roles.role(:web).asset_volume_args + assert_nil config_with_roles.role(:workers).asset_volume_args + assert_nil config_with_roles.role(:web).asset_path + assert_nil config_with_roles.role(:workers).asset_path + assert_not config_with_roles.role(:web).assets? + assert_not config_with_roles.role(:workers).assets? config_with_assets = Kamal::Configuration.new(@deploy_with_roles.dup.tap { |c| c[:asset_path] = "foo" @@ -258,17 +253,26 @@ class ConfigurationRoleTest < ActiveSupport::TestCase test "asset extracted path" do ENV["VERSION"] = "12345" - assert_equal ".kamal/assets/extracted/app-web-12345", @config_with_roles.role(:web).asset_extracted_path - assert_equal ".kamal/assets/extracted/app-workers-12345", @config_with_roles.role(:workers).asset_extracted_path + assert_equal ".kamal/assets/extracted/app-web-12345", config_with_roles.role(:web).asset_extracted_path + assert_equal ".kamal/assets/extracted/app-workers-12345", config_with_roles.role(:workers).asset_extracted_path ensure ENV.delete("VERSION") end test "asset volume path" do ENV["VERSION"] = "12345" - assert_equal ".kamal/assets/volumes/app-web-12345", @config_with_roles.role(:web).asset_volume_path - assert_equal ".kamal/assets/volumes/app-workers-12345", @config_with_roles.role(:workers).asset_volume_path + assert_equal ".kamal/assets/volumes/app-web-12345", config_with_roles.role(:web).asset_volume_path + assert_equal ".kamal/assets/volumes/app-workers-12345", config_with_roles.role(:workers).asset_volume_path ensure ENV.delete("VERSION") end + + private + def config + Kamal::Configuration.new(@deploy) + end + + def config_with_roles + Kamal::Configuration.new(@deploy_with_roles) + end end diff --git a/test/integration/accessory_test.rb b/test/integration/accessory_test.rb index 8c23703b..bc2fb378 100644 --- a/test/integration/accessory_test.rb +++ b/test/integration/accessory_test.rb @@ -2,8 +2,6 @@ require_relative "integration_test" class AccessoryTest < IntegrationTest test "boot, stop, start, restart, logs, remove" do - kamal :envify - kamal :accessory, :boot, :busybox assert_accessory_running :busybox @@ -21,8 +19,6 @@ class AccessoryTest < IntegrationTest kamal :accessory, :remove, :busybox, "-y" assert_accessory_not_running :busybox - - kamal :env, :delete end private diff --git a/test/integration/app_test.rb b/test/integration/app_test.rb index 9824ce0d..b7dcdc34 100644 --- a/test/integration/app_test.rb +++ b/test/integration/app_test.rb @@ -2,8 +2,6 @@ require_relative "integration_test" class AppTest < IntegrationTest test "stop, start, boot, logs, images, containers, exec, remove" do - kamal :envify - kamal :deploy assert_app_is_up diff --git a/test/integration/broken_deploy_test.rb b/test/integration/broken_deploy_test.rb index 5ab24f55..77f0ff96 100644 --- a/test/integration/broken_deploy_test.rb +++ b/test/integration/broken_deploy_test.rb @@ -4,8 +4,6 @@ class BrokenDeployTest < IntegrationTest test "deploying a bad image" do @app = "app_with_roles" - kamal :envify - first_version = latest_app_version kamal :deploy diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml index 8f095f0f..6df0935d 100644 --- a/test/integration/docker-compose.yml +++ b/test/integration/docker-compose.yml @@ -29,6 +29,8 @@ services: context: docker/registry environment: - REGISTRY_HTTP_ADDR=0.0.0.0:4443 + - REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt + - REGISTRY_HTTP_TLS_KEY=/certs/domain.key volumes: - shared:/shared - registry:/var/lib/registry/ diff --git a/test/integration/docker/deployer/Dockerfile b/test/integration/docker/deployer/Dockerfile index a809a6ed..269f78b0 100644 --- a/test/integration/docker/deployer/Dockerfile +++ b/test/integration/docker/deployer/Dockerfile @@ -22,11 +22,12 @@ COPY app_with_roles/ app_with_roles/ RUN rm -rf /root/.ssh RUN ln -s /shared/ssh /root/.ssh +RUN mkdir -p /etc/docker/certs.d/registry:4443 && ln -s /shared/certs/domain.crt /etc/docker/certs.d/registry:4443/ca.crt RUN git config --global user.email "deployer@example.com" RUN git config --global user.name "Deployer" -RUN cd app && git init && echo ".env" >> .gitignore && git add . && git commit -am "Initial version" -RUN cd app_with_roles && git init && echo ".env" >> .gitignore && git add . && git commit -am "Initial version" +RUN cd app && git init && git add . && git commit -am "Initial version" +RUN cd app_with_roles && git init && git add . && git commit -am "Initial version" HEALTHCHECK --interval=1s CMD pgrep sleep diff --git a/test/integration/docker/deployer/app/.kamal/secrets b/test/integration/docker/deployer/app/.kamal/secrets new file mode 100644 index 00000000..429499ce --- /dev/null +++ b/test/integration/docker/deployer/app/.kamal/secrets @@ -0,0 +1,4 @@ +SECRETS=$(kamal secrets fetch --adapter test --account test INTERPOLATED_SECRET1 INTERPOLATED_SECRET2 INTERPOLATED_中文) +INTERPOLATED_SECRET1=$(kamal secrets extract INTERPOLATED_SECRET1 ${SECRETS}) +INTERPOLATED_SECRET2=$(kamal secrets extract INTERPOLATED_SECRET2 ${SECRETS}) +INTERPOLATED_SECRET3=$(kamal secrets extract INTERPOLATED_中文 ${SECRETS}) diff --git a/test/integration/docker/deployer/app/.env.erb b/test/integration/docker/deployer/app/.kamal/secrets-common similarity index 100% rename from test/integration/docker/deployer/app/.env.erb rename to test/integration/docker/deployer/app/.kamal/secrets-common diff --git a/test/integration/docker/deployer/app/config/deploy.yml b/test/integration/docker/deployer/app/config/deploy.yml index 887de825..a85412ad 100644 --- a/test/integration/docker/deployer/app/config/deploy.yml +++ b/test/integration/docker/deployer/app/config/deploy.yml @@ -10,6 +10,9 @@ env: HOST_TOKEN: "${HOST_TOKEN}" secret: - SECRET_TOKEN + - INTERPOLATED_SECRET1 + - INTERPOLATED_SECRET2 + - INTERPOLATED_SECRET3 tags: tag1: CLEAR_TAG: tagged diff --git a/test/integration/docker/deployer/app_with_roles/.env.erb b/test/integration/docker/deployer/app_with_roles/.kamal/secrets similarity index 100% rename from test/integration/docker/deployer/app_with_roles/.env.erb rename to test/integration/docker/deployer/app_with_roles/.kamal/secrets diff --git a/test/integration/docker/deployer/boot.sh b/test/integration/docker/deployer/boot.sh index 20e1375d..77d6d1ea 100755 --- a/test/integration/docker/deployer/boot.sh +++ b/test/integration/docker/deployer/boot.sh @@ -1,5 +1,5 @@ #!/bin/bash -dockerd --max-concurrent-downloads 1 --insecure-registry registry:4443 & +dockerd --max-concurrent-downloads 1 & exec sleep infinity diff --git a/test/integration/docker/registry/boot.sh b/test/integration/docker/registry/boot.sh index 6eb8a518..895838f5 100755 --- a/test/integration/docker/registry/boot.sh +++ b/test/integration/docker/registry/boot.sh @@ -1,3 +1,5 @@ #!/bin/sh +while [ ! -f /certs/domain.crt ]; do sleep 1; done + exec /entrypoint.sh /etc/docker/registry/config.yml diff --git a/test/integration/docker/shared/Dockerfile b/test/integration/docker/shared/Dockerfile index 348fa4f3..f672fbe1 100644 --- a/test/integration/docker/shared/Dockerfile +++ b/test/integration/docker/shared/Dockerfile @@ -10,6 +10,8 @@ RUN mkdir ssh && \ COPY registry-dns.conf . COPY boot.sh . +RUN mkdir certs && openssl req -newkey rsa:4096 -nodes -sha256 -keyout certs/domain.key -x509 -days 365 -out certs/domain.crt -subj '/CN=registry' -extensions EXT -config registry-dns.conf + HEALTHCHECK --interval=1s CMD pgrep sleep CMD ["./boot.sh"] diff --git a/test/integration/docker/vm/Dockerfile b/test/integration/docker/vm/Dockerfile index 014794b0..584a7b6a 100644 --- a/test/integration/docker/vm/Dockerfile +++ b/test/integration/docker/vm/Dockerfile @@ -5,6 +5,7 @@ WORKDIR /work RUN apt-get update --fix-missing && apt-get -y install openssh-client openssh-server docker.io RUN mkdir /root/.ssh && ln -s /shared/ssh/id_rsa.pub /root/.ssh/authorized_keys +RUN mkdir -p /etc/docker/certs.d/registry:4443 && ln -s /shared/certs/domain.crt /etc/docker/certs.d/registry:4443/ca.crt RUN echo "HOST_TOKEN=abcd" >> /etc/environment diff --git a/test/integration/docker/vm/boot.sh b/test/integration/docker/vm/boot.sh index ecdbdb3c..681a8a4e 100755 --- a/test/integration/docker/vm/boot.sh +++ b/test/integration/docker/vm/boot.sh @@ -4,6 +4,6 @@ while [ ! -f /root/.ssh/authorized_keys ]; do echo "Waiting for ssh keys"; sleep service ssh restart -dockerd --max-concurrent-downloads 1 --insecure-registry registry:4443 & +dockerd --max-concurrent-downloads 1 & exec sleep infinity diff --git a/test/integration/lock_test.rb b/test/integration/lock_test.rb index db086251..4a53ba51 100644 --- a/test/integration/lock_test.rb +++ b/test/integration/lock_test.rb @@ -2,8 +2,6 @@ require_relative "integration_test" class LockTest < IntegrationTest test "acquire, release, status" do - kamal :envify - kamal :lock, :acquire, "-m 'Integration Tests'" status = kamal :lock, :status, capture: true diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb index a1e81493..b58aeeb9 100644 --- a/test/integration/main_test.rb +++ b/test/integration/main_test.rb @@ -1,11 +1,7 @@ require_relative "integration_test" class MainTest < IntegrationTest - test "envify, deploy, redeploy, rollback, details and audit" do - kamal :envify - assert_env_files - remove_local_env_file - + test "deploy, redeploy, rollback, details and audit" do first_version = latest_app_version assert_app_is_down @@ -37,16 +33,11 @@ class MainTest < IntegrationTest audit = kamal :audit, capture: true assert_match /Booted app version #{first_version}.*Booted app version #{second_version}.*Booted app version #{first_version}.*/m, audit - - kamal :env, :delete - assert_no_remote_env_file end test "app with roles" do @app = "app_with_roles" - kamal :envify - version = latest_app_version assert_app_is_down @@ -77,7 +68,7 @@ class MainTest < IntegrationTest assert_equal "app-#{version}", config[:service_with_version] assert_equal [], config[:volume_args] assert_equal({ user: "root", port: 22, keepalive: true, keepalive_interval: 30, log_level: :fatal }, config[:ssh_options]) - assert_equal({ "driver" => "docker", "arch" => "amd64", "args" => { "COMMIT_SHA" => version } }, config[:builder]) + assert_equal({ "driver" => "docker", "arch" => "#{Kamal::Utils.docker_arch}", "args" => { "COMMIT_SHA" => version } }, config[:builder]) assert_equal [ "--log-opt", "max-size=\"10m\"" ], config[:logging] assert_equal({ "cmd"=>"wget -qO- http://localhost > /dev/null || exit 1", "interval"=>"1s", "max_attempts"=>3, "port"=>3000, "path"=>"/up", "cord"=>"/tmp/kamal-cord", "log_lines"=>50 }, config[:healthcheck]) end @@ -103,7 +94,6 @@ class MainTest < IntegrationTest kamal :remove, "-y" assert_no_images_or_containers - kamal :envify kamal :setup assert_images_and_containers @@ -112,18 +102,17 @@ class MainTest < IntegrationTest end private - def assert_local_env_file(contents) - assert_equal contents, deployer_exec("cat .env", capture: true) - end - def assert_envs(version:) assert_env :CLEAR_TOKEN, "4321", version: version, vm: :vm1 assert_env :HOST_TOKEN, "abcd", version: version, vm: :vm1 assert_env :SECRET_TOKEN, "1234 with \"中文\"", version: version, vm: :vm1 assert_no_env :CLEAR_TAG, version: version, vm: :vm1 - assert_no_env :SECRET_TAG, version: version, vm: :vm11 + assert_no_env :SECRET_TAG, version: version, vm: :vm1 assert_env :CLEAR_TAG, "tagged", version: version, vm: :vm2 assert_env :SECRET_TAG, "TAGME", version: version, vm: :vm2 + assert_env :INTERPOLATED_SECRET1, "1TERCES_DETALOPRETNI", version: version, vm: :vm2 + assert_env :INTERPOLATED_SECRET2, "2TERCES_DETALOPRETNI", version: version, vm: :vm2 + assert_env :INTERPOLATED_SECRET3, "文中_DETALOPRETNI", version: version, vm: :vm2 end def assert_env(key, value, vm:, version:) @@ -136,24 +125,6 @@ class MainTest < IntegrationTest end end - def assert_env_files - assert_local_env_file "SECRET_TOKEN='1234 with \"中文\"'\nSECRET_TAG='TAGME'" - assert_remote_env_file "SECRET_TOKEN=1234 with \"中文\"", vm: :vm1 - assert_remote_env_file "SECRET_TOKEN=1234 with \"中文\"\nSECRET_TAG=TAGME", vm: :vm2 - end - - def remove_local_env_file - deployer_exec("rm .env") - end - - def assert_remote_env_file(contents, vm:) - assert_equal contents, docker_compose("exec #{vm} cat /root/.kamal/env/roles/app-web.env", capture: true) - end - - def assert_no_remote_env_file - assert_equal "nofile", docker_compose("exec vm1 stat /root/.kamal/env/roles/app-web.env 2> /dev/null || echo nofile", capture: true) - end - def assert_accumulated_assets(*versions) versions.each do |version| assert_equal "200", Net::HTTP.get_response(URI.parse("http://localhost:12345/versions/#{version}")).code diff --git a/test/integration/traefik_test.rb b/test/integration/traefik_test.rb index d2aa2a97..48f9ea02 100644 --- a/test/integration/traefik_test.rb +++ b/test/integration/traefik_test.rb @@ -2,8 +2,6 @@ require_relative "integration_test" class TraefikTest < IntegrationTest test "boot, reboot, stop, start, restart, logs, remove" do - kamal :envify - kamal :traefik, :boot assert_traefik_running @@ -46,8 +44,6 @@ class TraefikTest < IntegrationTest kamal :traefik, :remove assert_traefik_not_running - - kamal :env, :delete end private diff --git a/test/secrets/bitwarden_adapter_test.rb b/test/secrets/bitwarden_adapter_test.rb new file mode 100644 index 00000000..ff3f2a1c --- /dev/null +++ b/test/secrets/bitwarden_adapter_test.rb @@ -0,0 +1,211 @@ +require "test_helper" + +class BitwardenAdapterTest < SecretAdapterTestCase + test "fetch" do + stub_unlocked + stub_ticks.with("bw sync").returns("") + stub_mypassword + + json = JSON.parse(shellunescape(run_command("fetch", "mypassword"))) + + expected_json = { "mypassword"=>"secret123" } + + assert_equal expected_json, json + end + + test "fetch with from" do + stub_unlocked + stub_ticks.with("bw sync").returns("") + stub_myitem + + json = JSON.parse(shellunescape(run_command("fetch", "--from", "myitem", "field1", "field2", "field3"))) + + expected_json = { + "myitem/field1"=>"secret1", "myitem/field2"=>"blam", "myitem/field3"=>"fewgrwjgk" + } + + assert_equal expected_json, json + end + + test "fetch with multiple items" do + stub_unlocked + + stub_ticks.with("bw sync").returns("") + stub_mypassword + stub_myitem + + stub_ticks + .with("bw get item myitem2") + .returns(<<~JSON) + { + "passwordHistory":null, + "revisionDate":"2024-08-29T13:46:53.343Z", + "creationDate":"2024-08-29T12:02:31.156Z", + "deletedDate":null, + "object":"item", + "id":"aaaaaaaa-cccc-eeee-0000-222222222222", + "organizationId":null, + "folderId":null, + "type":1, + "reprompt":0, + "name":"myitem2", + "notes":null, + "favorite":false, + "fields":[ + {"name":"field3","value":"fewgrwjgk","type":1,"linkedId":null} + ], + "login":{"fido2Credentials":[],"uris":[],"username":null,"password":null,"totp":null,"passwordRevisionDate":null},"collectionIds":[] + } + JSON + + + json = JSON.parse(shellunescape(run_command("fetch", "mypassword", "myitem/field1", "myitem/field2", "myitem2/field3"))) + + expected_json = { + "mypassword"=>"secret123", "myitem/field1"=>"secret1", "myitem/field2"=>"blam", "myitem2/field3"=>"fewgrwjgk" + } + + assert_equal expected_json, json + end + + test "fetch unauthenticated" do + stub_ticks + .with("bw status") + .returns( + '{"serverUrl":null,"lastSync":null,"status":"unauthenticated"}', + '{"serverUrl":null,"lastSync":"2024-09-04T10:11:12.433Z","userEmail":"email@example.com","userId":"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee","status":"locked"}', + '{"serverUrl":null,"lastSync":"2024-09-04T10:11:12.433Z","userEmail":"email@example.com","userId":"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee","status":"unlocked"}' + ) + + stub_ticks.with("bw login email@example.com").returns("1234567890") + stub_ticks.with("bw unlock --raw").returns("") + stub_ticks.with("bw sync").returns("") + stub_mypassword + + json = JSON.parse(shellunescape(run_command("fetch", "mypassword"))) + + expected_json = { "mypassword"=>"secret123" } + + assert_equal expected_json, json + end + + test "fetch locked" do + stub_ticks + .with("bw status") + .returns( + '{"serverUrl":null,"lastSync":"2024-09-04T10:11:12.433Z","userEmail":"email@example.com","userId":"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee","status":"locked"}' + ) + + stub_ticks + .with("bw status") + .returns( + '{"serverUrl":null,"lastSync":"2024-09-04T10:11:12.433Z","userEmail":"email@example.com","userId":"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee","status":"unlocked"}' + ) + + stub_ticks.with("bw login email@example.com").returns("1234567890") + stub_ticks.with("bw unlock --raw").returns("") + stub_ticks.with("bw sync").returns("") + stub_mypassword + + json = JSON.parse(shellunescape(run_command("fetch", "mypassword"))) + + expected_json = { "mypassword"=>"secret123" } + + assert_equal expected_json, json + end + + test "fetch locked with session" do + stub_ticks + .with("bw status") + .returns( + '{"serverUrl":null,"lastSync":"2024-09-04T10:11:12.433Z","userEmail":"email@example.com","userId":"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee","status":"locked"}' + ) + + stub_ticks + .with("BW_SESSION=0987654321 bw status") + .returns( + '{"serverUrl":null,"lastSync":"2024-09-04T10:11:12.433Z","userEmail":"email@example.com","userId":"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee","status":"unlocked"}' + ) + + stub_ticks.with("bw login email@example.com").returns("1234567890") + stub_ticks.with("bw unlock --raw").returns("0987654321") + stub_ticks.with("BW_SESSION=0987654321 bw sync").returns("") + stub_mypassword(session: "0987654321") + + json = JSON.parse(shellunescape(run_command("fetch", "mypassword"))) + + expected_json = { "mypassword"=>"secret123" } + + assert_equal expected_json, json + end + + private + def run_command(*command) + stdouted do + Kamal::Cli::Secrets.start \ + [ *command, + "-c", "test/fixtures/deploy_with_accessories.yml", + "--adapter", "bitwarden", + "--account", "email@example.com" ] + end + end + + def stub_unlocked + stub_ticks + .with("bw status") + .returns(<<~JSON) + {"serverUrl":null,"lastSync":"2024-09-04T10:11:12.433Z","userEmail":"email@example.com","userId":"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee","status":"unlocked"} + JSON + end + + def stub_mypassword(session: nil) + stub_ticks + .with("#{"BW_SESSION=#{session} " if session}bw get item mypassword") + .returns(<<~JSON) + { + "passwordHistory":null, + "revisionDate":"2024-08-29T13:46:53.343Z", + "creationDate":"2024-08-29T12:02:31.156Z", + "deletedDate":null, + "object":"item", + "id":"aaaaaaaa-cccc-eeee-0000-222222222222", + "organizationId":null, + "folderId":null, + "type":1, + "reprompt":0, + "name":"mypassword", + "notes":null, + "favorite":false, + "login":{"fido2Credentials":[],"uris":[],"username":null,"password":"secret123","totp":null,"passwordRevisionDate":null},"collectionIds":[] + } + JSON + end + + def stub_myitem + stub_ticks + .with("bw get item myitem") + .returns(<<~JSON) + { + "passwordHistory":null, + "revisionDate":"2024-08-29T13:46:53.343Z", + "creationDate":"2024-08-29T12:02:31.156Z", + "deletedDate":null, + "object":"item", + "id":"aaaaaaaa-cccc-eeee-0000-222222222222", + "organizationId":null, + "folderId":null, + "type":1, + "reprompt":0, + "name":"myitem", + "notes":null, + "favorite":false, + "fields":[ + {"name":"field1","value":"secret1","type":1,"linkedId":null}, + {"name":"field2","value":"blam","type":1,"linkedId":null}, + {"name":"field3","value":"fewgrwjgk","type":1,"linkedId":null} + ], + "login":{"fido2Credentials":[],"uris":[],"username":null,"password":null,"totp":null,"passwordRevisionDate":null},"collectionIds":[] + } + JSON + end +end diff --git a/test/secrets/dotenv_inline_command_substitution_test.rb b/test/secrets/dotenv_inline_command_substitution_test.rb new file mode 100644 index 00000000..3ae10ca9 --- /dev/null +++ b/test/secrets/dotenv_inline_command_substitution_test.rb @@ -0,0 +1,15 @@ +require "test_helper" + +class SecretsInlineCommandSubstitution < SecretAdapterTestCase + test "inlines kamal secrets commands" do + Kamal::Cli::Main.expects(:start).with { |command| command == [ "secrets", "fetch", "...", "--inline" ] }.returns("results") + substituted = Kamal::Secrets::Dotenv::InlineCommandSubstitution.call("FOO=$(kamal secrets fetch ...)", nil, overwrite: false) + assert_equal "FOO=results", substituted + end + + test "executes other commands" do + Kamal::Secrets::Dotenv::InlineCommandSubstitution.stubs(:`).with("blah").returns("results") + substituted = Kamal::Secrets::Dotenv::InlineCommandSubstitution.call("FOO=$(blah)", nil, overwrite: false) + assert_equal "FOO=results", substituted + end +end diff --git a/test/secrets/last_pass_adapter_test.rb b/test/secrets/last_pass_adapter_test.rb new file mode 100644 index 00000000..3801d486 --- /dev/null +++ b/test/secrets/last_pass_adapter_test.rb @@ -0,0 +1,152 @@ +require "test_helper" + +class LastPassAdapterTest < SecretAdapterTestCase + setup do + `true` # Ensure $? is 0 + end + + test "fetch" do + stub_ticks.with("lpass status --color never").returns("Logged in as email@example.com.") + + stub_ticks + .with("lpass show SECRET1 FOLDER1/FSECRET1 FOLDER1/FSECRET2 --json") + .returns(<<~JSON) + [ + { + "id": "1234567891234567891", + "name": "SECRET1", + "fullname": "SECRET1", + "username": "", + "password": "secret1", + "last_modified_gmt": "1724926054", + "last_touch": "1724926639", + "group": "", + "url": "", + "note": "" + }, + { + "id": "1234567891234567892", + "name": "FSECRET1", + "fullname": "FOLDER1/FSECRET1", + "username": "", + "password": "fsecret1", + "last_modified_gmt": "1724926084", + "last_touch": "1724926635", + "group": "Folder", + "url": "", + "note": "" + }, + { + "id": "1234567891234567893", + "name": "FSECRET2", + "fullname": "FOLDER1/FSECRET2", + "username": "", + "password": "fsecret2", + "last_modified_gmt": "1724926084", + "last_touch": "1724926635", + "group": "Folder", + "url": "", + "note": "" + } + ] + JSON + + json = JSON.parse(shellunescape(run_command("fetch", "SECRET1", "FOLDER1/FSECRET1", "FOLDER1/FSECRET2"))) + + expected_json = { + "SECRET1"=>"secret1", + "FOLDER1/FSECRET1"=>"fsecret1", + "FOLDER1/FSECRET2"=>"fsecret2" + } + + assert_equal expected_json, json + end + + test "fetch with from" do + stub_ticks.with("lpass status --color never").returns("Logged in as email@example.com.") + + stub_ticks + .with("lpass show FOLDER1/FSECRET1 FOLDER1/FSECRET2 --json") + .returns(<<~JSON) + [ + { + "id": "1234567891234567892", + "name": "FSECRET1", + "fullname": "FOLDER1/FSECRET1", + "username": "", + "password": "fsecret1", + "last_modified_gmt": "1724926084", + "last_touch": "1724926635", + "group": "Folder", + "url": "", + "note": "" + }, + { + "id": "1234567891234567893", + "name": "FSECRET2", + "fullname": "FOLDER1/FSECRET2", + "username": "", + "password": "fsecret2", + "last_modified_gmt": "1724926084", + "last_touch": "1724926635", + "group": "Folder", + "url": "", + "note": "" + } + ] + JSON + + json = JSON.parse(shellunescape(run_command("fetch", "--from", "FOLDER1", "FSECRET1", "FSECRET2"))) + + expected_json = { + "FOLDER1/FSECRET1"=>"fsecret1", + "FOLDER1/FSECRET2"=>"fsecret2" + } + + assert_equal expected_json, json + end + + test "fetch with signin" do + stub_ticks_with("lpass status --color never", succeed: false).returns("Not logged in.") + stub_ticks_with("lpass login email@example.com", succeed: true).returns("") + stub_ticks.with("lpass show SECRET1 --json").returns(single_item_json) + + json = JSON.parse(shellunescape(run_command("fetch", "SECRET1"))) + + expected_json = { + "SECRET1"=>"secret1" + } + + assert_equal expected_json, json + end + + private + def run_command(*command) + stdouted do + Kamal::Cli::Secrets.start \ + [ *command, + "-c", "test/fixtures/deploy_with_accessories.yml", + "--adapter", "lastpass", + "--account", "email@example.com" ] + end + end + + def single_item_json + <<~JSON + [ + { + "id": "1234567891234567891", + "name": "SECRET1", + "fullname": "SECRET1", + "username": "", + "password": "secret1", + "last_modified_gmt": "1724926054", + "last_touch": "1724926639", + "group": "", + "url": "", + "note": "" + } + ] + JSON + end +end diff --git a/test/secrets/one_password_adapter_test.rb b/test/secrets/one_password_adapter_test.rb new file mode 100644 index 00000000..59ad511d --- /dev/null +++ b/test/secrets/one_password_adapter_test.rb @@ -0,0 +1,177 @@ +require "test_helper" + +class SecretsOnePasswordAdapterTest < SecretAdapterTestCase + test "fetch" do + stub_ticks.with("op account get --account myaccount 2> /dev/null") + + stub_ticks + .with("op item get myitem --vault \"myvault\" --fields \"label=section.SECRET1,label=section.SECRET2,label=section2.SECRET3\" --format \"json\" --account \"myaccount\"") + .returns(<<~JSON) + [ + { + "id": "aaaaaaaaaaaaaaaaaaaaaaaaaa", + "section": { + "id": "cccccccccccccccccccccccccc", + "label": "section" + }, + "type": "CONCEALED", + "label": "SECRET1", + "value": "VALUE1", + "reference": "op://myvault/myitem/section/SECRET1" + }, + { + "id": "bbbbbbbbbbbbbbbbbbbbbbbbbb", + "section": { + "id": "dddddddddddddddddddddddddd", + "label": "section" + }, + "type": "CONCEALED", + "label": "SECRET2", + "value": "VALUE2", + "reference": "op://myvault/myitem/section/SECRET2" + }, + { + "id": "bbbbbbbbbbbbbbbbbbbbbbbbbb", + "section": { + "id": "dddddddddddddddddddddddddd", + "label": "section2" + }, + "type": "CONCEALED", + "label": "SECRET3", + "value": "VALUE3", + "reference": "op://myvault/myitem/section2/SECRET3" + } + ] + JSON + + json = JSON.parse(shellunescape(run_command("fetch", "--from", "op://myvault/myitem", "section/SECRET1", "section/SECRET2", "section2/SECRET3"))) + + expected_json = { + "myvault/myitem/section/SECRET1"=>"VALUE1", + "myvault/myitem/section/SECRET2"=>"VALUE2", + "myvault/myitem/section2/SECRET3"=>"VALUE3" + } + + assert_equal expected_json, json + end + + test "fetch with multiple items" do + stub_ticks.with("op account get --account myaccount 2> /dev/null") + + stub_ticks + .with("op item get myitem --vault \"myvault\" --fields \"label=section.SECRET1,label=section.SECRET2\" --format \"json\" --account \"myaccount\"") + .returns(<<~JSON) + [ + { + "id": "aaaaaaaaaaaaaaaaaaaaaaaaaa", + "section": { + "id": "cccccccccccccccccccccccccc", + "label": "section" + }, + "type": "CONCEALED", + "label": "SECRET1", + "value": "VALUE1", + "reference": "op://myvault/myitem/section/SECRET1" + }, + { + "id": "bbbbbbbbbbbbbbbbbbbbbbbbbb", + "section": { + "id": "dddddddddddddddddddddddddd", + "label": "section" + }, + "type": "CONCEALED", + "label": "SECRET2", + "value": "VALUE2", + "reference": "op://myvault/myitem/section/SECRET2" + } + ] + JSON + + stub_ticks + .with("op item get myitem2 --vault \"myvault\" --fields \"label=section2.SECRET3\" --format \"json\" --account \"myaccount\"") + .returns(<<~JSON) + { + "id": "aaaaaaaaaaaaaaaaaaaaaaaaaa", + "section": { + "id": "cccccccccccccccccccccccccc", + "label": "section" + }, + "type": "CONCEALED", + "label": "SECRET3", + "value": "VALUE3", + "reference": "op://myvault/myitem2/section/SECRET3" + } + JSON + + json = JSON.parse(shellunescape(run_command("fetch", "--from", "op://myvault", "myitem/section/SECRET1", "myitem/section/SECRET2", "myitem2/section2/SECRET3"))) + + expected_json = { + "myvault/myitem/section/SECRET1"=>"VALUE1", + "myvault/myitem/section/SECRET2"=>"VALUE2", + "myvault/myitem2/section/SECRET3"=>"VALUE3" + } + + assert_equal expected_json, json + end + + test "fetch with signin, no session" do + stub_ticks_with("op account get --account myaccount 2> /dev/null", succeed: false) + stub_ticks_with("op signin --account \"myaccount\" --force --raw", succeed: true).returns("") + + stub_ticks + .with("op item get myitem --vault \"myvault\" --fields \"label=section.SECRET1\" --format \"json\" --account \"myaccount\"") + .returns(single_item_json) + + json = JSON.parse(shellunescape(run_command("fetch", "--from", "op://myvault/myitem", "section/SECRET1"))) + + expected_json = { + "myvault/myitem/section/SECRET1"=>"VALUE1" + } + + assert_equal expected_json, json + end + + test "fetch with signin and session" do + stub_ticks_with("op account get --account myaccount 2> /dev/null", succeed: false) + stub_ticks_with("op signin --account \"myaccount\" --force --raw", succeed: true).returns("1234567890") + + stub_ticks + .with("op item get myitem --vault \"myvault\" --fields \"label=section.SECRET1\" --format \"json\" --account \"myaccount\" --session \"1234567890\"") + .returns(single_item_json) + + json = JSON.parse(shellunescape(run_command("fetch", "--from", "op://myvault/myitem", "section/SECRET1"))) + + expected_json = { + "myvault/myitem/section/SECRET1"=>"VALUE1" + } + + assert_equal expected_json, json + end + + private + def run_command(*command) + stdouted do + Kamal::Cli::Secrets.start \ + [ *command, + "-c", "test/fixtures/deploy_with_accessories.yml", + "--adapter", "1password", + "--account", "myaccount" ] + end + end + + def single_item_json + <<~JSON + { + "id": "aaaaaaaaaaaaaaaaaaaaaaaaaa", + "section": { + "id": "cccccccccccccccccccccccccc", + "label": "section" + }, + "type": "CONCEALED", + "label": "SECRET1", + "value": "VALUE1", + "reference": "op://myvault/myitem/section/SECRET1" + } + JSON + end +end diff --git a/test/secrets_test.rb b/test/secrets_test.rb new file mode 100644 index 00000000..bb77a196 --- /dev/null +++ b/test/secrets_test.rb @@ -0,0 +1,34 @@ +require "test_helper" + +class SecretsTest < ActiveSupport::TestCase + test "fetch" do + with_test_secrets("secrets" => "SECRET=ABC") do + assert_equal "ABC", Kamal::Secrets.new["SECRET"] + end + end + + test "command interpolation" do + with_test_secrets("secrets" => "SECRET=$(echo ABC)") do + assert_equal "ABC", Kamal::Secrets.new["SECRET"] + end + end + + test "variable references" do + with_test_secrets("secrets" => "SECRET1=ABC\nSECRET2=${SECRET1}DEF") do + assert_equal "ABC", Kamal::Secrets.new["SECRET1"] + assert_equal "ABCDEF", Kamal::Secrets.new["SECRET2"] + end + end + + test "destinations" do + with_test_secrets("secrets.dest" => "SECRET=DEF", "secrets" => "SECRET=ABC", "secrets-common" => "SECRET=GHI\nSECRET2=JKL") do + assert_equal "ABC", Kamal::Secrets.new["SECRET"] + assert_equal "DEF", Kamal::Secrets.new(destination: "dest")["SECRET"] + assert_equal "GHI", Kamal::Secrets.new(destination: "nodest")["SECRET"] + + assert_equal "JKL", Kamal::Secrets.new["SECRET2"] + assert_equal "JKL", Kamal::Secrets.new(destination: "dest")["SECRET2"] + assert_equal "JKL", Kamal::Secrets.new(destination: "nodest")["SECRET2"] + end + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 5f7a25c4..f8f4f4e4 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -34,4 +34,53 @@ class ActiveSupport::TestCase def stderred capture(:stderr) { yield }.strip end + + def with_test_secrets(**files) + setup_test_secrets(**files) + yield + ensure + teardown_test_secrets + end + + def setup_test_secrets(**files) + @original_pwd = Dir.pwd + @secrets_tmpdir = Dir.mktmpdir + fixtures_dup = File.join(@secrets_tmpdir, "test") + FileUtils.mkdir_p(fixtures_dup) + FileUtils.cp_r("test/fixtures/", fixtures_dup) + + Dir.chdir(@secrets_tmpdir) + FileUtils.mkdir_p(".kamal") + Dir.chdir(".kamal") do + files.each do |filename, contents| + File.binwrite(filename.to_s, contents) + end + end + end + + def teardown_test_secrets + Dir.chdir(@original_pwd) + FileUtils.rm_rf(@secrets_tmpdir) + end +end + +class SecretAdapterTestCase < ActiveSupport::TestCase + setup do + `true` # Ensure $? is 0 + end + + private + def stub_ticks + Kamal::Secrets::Adapters::Base.any_instance.stubs(:`) + end + + def stub_ticks_with(command, succeed: true) + # Sneakily run `false`/`true` after a match to set $? to 1/0 + stub_ticks.with { |c| c == command && (succeed ? `true` : `false`) } + Kamal::Secrets::Adapters::Base.any_instance.stubs(:`) + end + + def shellunescape(string) + "\"#{string}\"".undump.gsub(/\\([{}])/, "\\1") + end end