diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 839bc4e6..dcd1b0ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ on: branches: - main pull_request: + workflow_dispatch: jobs: rubocop: name: RuboCop @@ -27,7 +28,7 @@ jobs: - "3.1" - "3.2" - "3.3" - - "3.4.0-preview2" + - "3.4" gemfile: - Gemfile - gemfiles/rails_edge.gemfile diff --git a/Gemfile.lock b/Gemfile.lock index c057d7f6..67963d62 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - kamal (2.3.0) + kamal (2.4.0) activesupport (>= 7.0) base64 (~> 0.2) bcrypt_pbkdf (~> 1.0) @@ -16,80 +16,87 @@ PATH GEM remote: https://rubygems.org/ specs: - actionpack (7.1.4.1) - actionview (= 7.1.4.1) - activesupport (= 7.1.4.1) + actionpack (8.0.0.1) + actionview (= 8.0.0.1) + activesupport (= 8.0.0.1) nokogiri (>= 1.8.5) - racc rack (>= 2.2.4) rack-session (>= 1.0.1) rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - actionview (7.1.4.1) - activesupport (= 7.1.4.1) + useragent (~> 0.16) + actionview (8.0.0.1) + activesupport (= 8.0.0.1) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activesupport (7.1.4.1) + activesupport (8.0.0.1) base64 + benchmark (>= 0.3) bigdecimal - concurrent-ruby (~> 1.0, >= 1.0.2) + concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) + logger (>= 1.4.2) minitest (>= 5.1) - mutex_m - tzinfo (~> 2.0) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) ast (2.4.2) base64 (0.2.0) bcrypt_pbkdf (1.1.1) bcrypt_pbkdf (1.1.1-arm64-darwin) bcrypt_pbkdf (1.1.1-x86_64-darwin) + benchmark (0.4.0) bigdecimal (3.1.8) builder (3.3.0) concurrent-ruby (1.3.4) connection_pool (2.4.1) crass (1.0.6) + date (3.4.1) debug (1.9.2) irb (~> 1.10) reline (>= 0.3.8) - dotenv (3.1.2) + dotenv (3.1.5) drb (2.2.1) ed25519 (1.3.0) erubi (1.13.0) i18n (1.14.6) concurrent-ruby (~> 1.0) - io-console (0.7.2) - irb (1.14.0) + io-console (0.8.0) + irb (1.14.2) rdoc (>= 4.0.0) reline (>= 0.4.2) - json (2.7.2) + json (2.9.0) language_server-protocol (3.17.0.3) - loofah (2.22.0) + logger (1.6.3) + loofah (2.23.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) - minitest (5.25.1) - mocha (2.4.5) + minitest (5.25.4) + mocha (2.7.1) ruby2_keywords (>= 0.0.5) - mutex_m (0.2.0) net-scp (4.0.0) net-ssh (>= 2.6.5, < 8.0.0) net-sftp (4.0.0) net-ssh (>= 5.0.0, < 8.0.0) net-ssh (7.3.0) - nokogiri (1.16.7-arm64-darwin) + nokogiri (1.17.2-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.7-x86_64-darwin) + nokogiri (1.17.2-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.7-x86_64-linux) + nokogiri (1.17.2-x86_64-linux) racc (~> 1.4) - parallel (1.25.1) - parser (3.3.4.0) + ostruct (0.6.1) + parallel (1.26.3) + parser (3.3.6.0) ast (~> 2.4.1) racc - psych (5.1.2) + psych (5.2.1) + date stringio racc (1.8.1) rack (3.1.8) @@ -97,55 +104,52 @@ GEM rack (>= 3.0.0) rack-test (2.1.0) rack (>= 1.3) - rackup (2.1.0) + rackup (2.2.1) rack (>= 3) - webrick (~> 1.8) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.6.0) + rails-html-sanitizer (1.6.2) loofah (~> 2.21) - nokogiri (~> 1.14) - railties (7.1.4.1) - actionpack (= 7.1.4.1) - activesupport (= 7.1.4.1) - irb + nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) + railties (8.0.0.1) + actionpack (= 8.0.0.1) + activesupport (= 8.0.0.1) + irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) thor (~> 1.0, >= 1.2.2) zeitwerk (~> 2.6) rainbow (3.1.1) rake (13.2.1) - rdoc (6.7.0) + rdoc (6.8.1) psych (>= 4.0.0) - regexp_parser (2.9.2) - reline (0.5.9) + regexp_parser (2.9.3) + reline (0.5.12) io-console (~> 0.5) - rexml (3.3.9) - rubocop (1.65.1) + rubocop (1.69.2) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 2.4, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.31.1, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.36.2, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.32.0) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.36.2) parser (>= 3.3.1.0) - rubocop-minitest (0.35.1) + rubocop-minitest (0.36.0) rubocop (>= 1.61, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) - rubocop-performance (1.21.1) + rubocop-performance (1.23.0) rubocop (>= 1.48.1, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) - rubocop-rails (2.25.1) + rubocop-rails (2.27.0) activesupport (>= 4.2.0) rack (>= 1.1) - rubocop (>= 1.33.0, < 2.0) + rubocop (>= 1.52.0, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) rubocop-rails-omakase (1.0.0) rubocop @@ -154,17 +158,22 @@ GEM rubocop-rails ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) - sshkit (1.23.0) + securerandom (0.4.0) + sshkit (1.23.2) base64 net-scp (>= 1.1.2) net-sftp (>= 2.1.2) net-ssh (>= 2.8.0) - stringio (3.1.1) - thor (1.3.1) + ostruct + stringio (3.1.2) + thor (1.3.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - unicode-display_width (2.5.0) - webrick (1.8.2) + unicode-display_width (3.1.2) + unicode-emoji (~> 4.0, >= 4.0.4) + unicode-emoji (4.0.4) + uri (1.0.2) + useragent (0.16.11) zeitwerk (2.7.1) PLATFORMS diff --git a/lib/kamal/cli.rb b/lib/kamal/cli.rb index dc35c403..769b4a17 100644 --- a/lib/kamal/cli.rb +++ b/lib/kamal/cli.rb @@ -2,6 +2,7 @@ module Kamal::Cli class BootError < StandardError; end class HookError < StandardError; end class LockError < StandardError; end + class DependencyError < StandardError; end end # SSHKit uses instance eval, so we need a global const for ergonomics diff --git a/lib/kamal/cli/accessory.rb b/lib/kamal/cli/accessory.rb index 4de8832b..c95cbb1e 100644 --- a/lib/kamal/cli/accessory.rb +++ b/lib/kamal/cli/accessory.rb @@ -162,7 +162,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)" option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server" option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)" - option :grep_options, aliases: "-o", desc: "Additional options supplied to grep" + option :grep_options, desc: "Additional options supplied to grep" option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)" option :skip_timestamps, type: :boolean, aliases: "-T", desc: "Skip appending timestamps to logging output" def logs(name) @@ -292,7 +292,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base def prepare(name) with_accessory(name) do |accessory, hosts| on(hosts) do - execute *KAMAL.registry.login + execute *KAMAL.registry.login(registry_config: accessory.registry) execute *KAMAL.docker.create_network rescue SSHKit::Command::Failed => e raise unless e.message.include?("already exists") diff --git a/lib/kamal/cli/app.rb b/lib/kamal/cli/app.rb index 74b7b4df..fb665af9 100644 --- a/lib/kamal/cli/app.rb +++ b/lib/kamal/cli/app.rb @@ -94,9 +94,15 @@ class Kamal::Cli::App < Kamal::Cli::Base option :interactive, aliases: "-i", type: :boolean, default: false, desc: "Execute command over ssh for an interactive shell (use for console/bash)" option :reuse, type: :boolean, default: false, desc: "Reuse currently running container instead of starting a new one" option :env, aliases: "-e", type: :hash, desc: "Set environment variables for the command" + option :detach, type: :boolean, default: false, desc: "Execute command in a detached container" def exec(*cmd) + if (incompatible_options = [ :interactive, :reuse ].select { |key| options[:detach] && options[key] }.presence) + raise ArgumentError, "Detach is not compatible with #{incompatible_options.join(" or ")}" + end + cmd = Kamal::Utils.join_commands(cmd) env = options[:env] + detach = options[:detach] case when options[:interactive] && options[:reuse] say "Get current version of running container...", :magenta unless options[:version] @@ -138,7 +144,7 @@ class Kamal::Cli::App < Kamal::Cli::Base roles.each do |role| execute *KAMAL.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug - puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).execute_in_new_container(cmd, env: env)) + puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).execute_in_new_container(cmd, env: env, detach: detach)) end end end @@ -186,15 +192,17 @@ class Kamal::Cli::App < Kamal::Cli::Base option :since, aliases: "-s", desc: "Show lines since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)" option :lines, type: :numeric, aliases: "-n", desc: "Number of lines to show from each server" option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)" - option :grep_options, aliases: "-o", desc: "Additional options supplied to grep" + option :grep_options, desc: "Additional options supplied to grep" option :follow, aliases: "-f", desc: "Follow log on primary server (or specific host set by --hosts)" option :skip_timestamps, type: :boolean, aliases: "-T", desc: "Skip appending timestamps to logging output" + option :container_id, desc: "Docker container ID to fetch logs" def logs # FIXME: Catch when app containers aren't running grep = options[:grep] grep_options = options[:grep_options] since = options[:since] + container_id = options[:container_id] timestamps = !options[:skip_timestamps] if options[:follow] @@ -207,8 +215,8 @@ class Kamal::Cli::App < Kamal::Cli::Base role = KAMAL.roles_on(KAMAL.primary_host).first app = KAMAL.app(role: role, host: host) - info app.follow_logs(host: KAMAL.primary_host, timestamps: timestamps, lines: lines, grep: grep, grep_options: grep_options) - exec app.follow_logs(host: KAMAL.primary_host, timestamps: timestamps, lines: lines, grep: grep, grep_options: grep_options) + info app.follow_logs(host: KAMAL.primary_host, container_id: container_id, timestamps: timestamps, lines: lines, grep: grep, grep_options: grep_options) + exec app.follow_logs(host: KAMAL.primary_host, container_id: container_id, timestamps: timestamps, lines: lines, grep: grep, grep_options: grep_options) end else lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set @@ -218,7 +226,7 @@ class Kamal::Cli::App < Kamal::Cli::Base roles.each do |role| begin - puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).logs(timestamps: timestamps, since: since, lines: lines, grep: grep, grep_options: grep_options)) + puts_by_host host, capture_with_info(*KAMAL.app(role: role, host: host).logs(container_id: container_id, timestamps: timestamps, since: since, lines: lines, grep: grep, grep_options: grep_options)) rescue SSHKit::Command::Failed puts_by_host host, "Nothing found" end diff --git a/lib/kamal/cli/app/boot.rb b/lib/kamal/cli/app/boot.rb index fd330c71..8ddcaac3 100644 --- a/lib/kamal/cli/app/boot.rb +++ b/lib/kamal/cli/app/boot.rb @@ -45,7 +45,7 @@ class Kamal::Cli::App::Boot def start_new_version audit "Booted app version #{version}" - hostname = "#{host.to_s[0...51].gsub(/\.+$/, '')}-#{SecureRandom.hex(6)}" + hostname = "#{host.to_s[0...51].chomp(".")}-#{SecureRandom.hex(6)}" execute *app.ensure_env_directory upload! role.secrets_io(host), role.secrets_path, mode: "0600" @@ -91,7 +91,7 @@ class Kamal::Cli::App::Boot if barrier.close info "First #{KAMAL.primary_role} container is unhealthy on #{host}, not booting any other roles" begin - error capture_with_info(*app.logs(version: version)) + error capture_with_info(*app.logs(container_id: app.container_id_for_version(version))) error capture_with_info(*app.container_health_log(version: version)) rescue SSHKit::Command::Failed error "Could not fetch logs for #{version}" diff --git a/lib/kamal/cli/base.rb b/lib/kamal/cli/base.rb index 4aebfd90..8dde4752 100644 --- a/lib/kamal/cli/base.rb +++ b/lib/kamal/cli/base.rb @@ -5,7 +5,7 @@ module Kamal::Cli class Base < Thor include SSHKit::DSL - def self.exit_on_failure?() false end + def self.exit_on_failure?() true end def self.dynamic_command_class() Kamal::Cli::Alias::Command end class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging" @@ -30,7 +30,8 @@ module Kamal::Cli else super end - initialize_commander unless KAMAL.configured? + + initialize_commander unless config[:invoked_via_subcommand] end private @@ -194,5 +195,19 @@ module Kamal::Cli ENV.clear ENV.update(current_env) end + + def ensure_docker_installed + run_locally do + begin + execute *KAMAL.builder.ensure_docker_installed + rescue SSHKit::Command::Failed => e + error = e.message =~ /command not found/ ? + "Docker is not installed locally" : + "Docker buildx plugin is not installed locally" + + raise DependencyError, error + end + end + end end end diff --git a/lib/kamal/cli/build.rb b/lib/kamal/cli/build.rb index 53ecb0bb..8897e2ae 100644 --- a/lib/kamal/cli/build.rb +++ b/lib/kamal/cli/build.rb @@ -13,7 +13,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base def push cli = self - verify_local_dependencies + ensure_docker_installed run_hook "pre-build" uncommitted_changes = Kamal::Git.uncommitted_changes @@ -109,20 +109,6 @@ class Kamal::Cli::Build < Kamal::Cli::Base end private - def verify_local_dependencies - run_locally do - begin - execute *KAMAL.builder.ensure_local_dependencies_installed - rescue SSHKit::Command::Failed => e - build_error = e.message =~ /command not found/ ? - "Docker is not installed locally" : - "Docker buildx plugin is not installed locally" - - raise BuildError, build_error - end - end - end - def connect_to_remote_host(remote_host) remote_uri = URI.parse(remote_host) if remote_uri.scheme == "ssh" diff --git a/lib/kamal/cli/main.rb b/lib/kamal/cli/main.rb index 5fdb5469..2fae36e8 100644 --- a/lib/kamal/cli/main.rb +++ b/lib/kamal/cli/main.rb @@ -9,15 +9,14 @@ class Kamal::Cli::Main < Kamal::Cli::Base say "Ensure Docker is installed...", :magenta invoke "kamal:cli:server:bootstrap", [], invoke_options - invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options - deploy + deploy(boot_accessories: true) end end end desc "deploy", "Deploy app to servers" option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push" - def deploy + def deploy(boot_accessories: false) runtime = print_runtime do invoke_options = deploy_options @@ -38,6 +37,8 @@ class Kamal::Cli::Main < Kamal::Cli::Base say "Ensure kamal-proxy is running...", :magenta invoke "kamal:cli:proxy:boot", [], invoke_options + invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options if boot_accessories + say "Detect stale containers...", :magenta invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true) diff --git a/lib/kamal/cli/proxy.rb b/lib/kamal/cli/proxy.rb index d0e9ba2b..f73177ad 100644 --- a/lib/kamal/cli/proxy.rb +++ b/lib/kamal/cli/proxy.rb @@ -23,6 +23,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base desc "boot_config ", "Manage kamal-proxy boot configuration" option :publish, type: :boolean, default: true, desc: "Publish the proxy ports on the host" + option :publish_host_ip, type: :string, repeatable: true, default: nil, desc: "Host IP address to bind HTTP/HTTPS traffic to. Defaults to all interfaces" option :http_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTP_PORT, desc: "HTTP port to publish on the host" option :https_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTPS_PORT, desc: "HTTPS port to publish on the host" option :log_max_size, type: :string, default: Kamal::Configuration::PROXY_LOG_MAX_SIZE, desc: "Max size of proxy logs" @@ -31,7 +32,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base case subcommand when "set" boot_options = [ - *(KAMAL.config.proxy_publish_args(options[:http_port], options[:https_port]) if options[:publish]), + *(KAMAL.config.proxy_publish_args(options[:http_port], options[:https_port], options[:publish_host_ip]) if options[:publish]), *(KAMAL.config.proxy_logging_args(options[:log_max_size])), *options[:docker_options].map { |option| "--#{option}" } ] @@ -67,9 +68,6 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base execute *KAMAL.auditor.record("Rebooted proxy"), verbosity: :debug execute *KAMAL.registry.login - "Stopping and removing Traefik on #{host}, if running..." - execute *KAMAL.proxy.cleanup_traefik - "Stopping and removing kamal-proxy on #{host}, if running..." execute *KAMAL.proxy.stop, raise_on_non_zero_exit: false execute *KAMAL.proxy.remove_container diff --git a/lib/kamal/cli/registry.rb b/lib/kamal/cli/registry.rb index 9d5d9d93..2fbdba1d 100644 --- a/lib/kamal/cli/registry.rb +++ b/lib/kamal/cli/registry.rb @@ -3,6 +3,8 @@ class Kamal::Cli::Registry < Kamal::Cli::Base option :skip_local, aliases: "-L", type: :boolean, default: false, desc: "Skip local login" option :skip_remote, aliases: "-R", type: :boolean, default: false, desc: "Skip remote login" def login + ensure_docker_installed + run_locally { execute *KAMAL.registry.login } unless options[:skip_local] on(KAMAL.hosts) { execute *KAMAL.registry.login } unless options[:skip_remote] end diff --git a/lib/kamal/cli/templates/deploy.yml b/lib/kamal/cli/templates/deploy.yml index 89104ad3..89c7a7c3 100644 --- a/lib/kamal/cli/templates/deploy.yml +++ b/lib/kamal/cli/templates/deploy.yml @@ -38,7 +38,7 @@ builder: arch: amd64 # Pass in additional build args needed for your Dockerfile. # args: - # RUBY_VERSION: <%= File.read('.ruby-version').strip %> + # RUBY_VERSION: <%= ENV["RBENV_VERSION"] || ENV["rvm_ruby_string"] || "#{RUBY_ENGINE}-#{RUBY_ENGINE_VERSION}" %> # Inject ENV variables into containers (secrets come from .kamal/secrets). # @@ -49,7 +49,7 @@ builder: # - RAILS_MASTER_KEY # Aliases are triggered with "bin/kamal ". You can overwrite arguments on invocation: -# "bin/kamal logs -r job" will tail logs from the first server in the job section. +# "bin/kamal app logs -r job" will tail logs from the first server in the job section. # # aliases: # shell: app exec --interactive --reuse "bash" diff --git a/lib/kamal/commander.rb b/lib/kamal/commander.rb index 1557df57..6a461276 100644 --- a/lib/kamal/commander.rb +++ b/lib/kamal/commander.rb @@ -76,11 +76,6 @@ class Kamal::Commander config.accessories&.collect(&:name) || [] end - def accessories_on(host) - config.accessories.select { |accessory| accessory.hosts.include?(host.to_s) }.map(&:name) - end - - def app(role: nil, host: nil) Kamal::Commands::App.new(config, role: role, host: host) end @@ -129,7 +124,6 @@ class Kamal::Commander config.aliases[name] end - def with_verbosity(level) old_level = self.verbosity diff --git a/lib/kamal/commands/accessory.rb b/lib/kamal/commands/accessory.rb index 281b8713..77ceb607 100644 --- a/lib/kamal/commands/accessory.rb +++ b/lib/kamal/commands/accessory.rb @@ -4,11 +4,10 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base attr_reader :accessory_config delegate :service_name, :image, :hosts, :port, :files, :directories, :cmd, :network_args, :publish_args, :env_args, :volume_args, :label_args, :option_args, - :secrets_io, :secrets_path, :env_directory, :proxy, :running_proxy?, + :secrets_io, :secrets_path, :env_directory, :proxy, :running_proxy?, :registry, to: :accessory_config delegate :proxy_container_name, to: :config - def initialize(config, name:) super(config) @accessory_config = config.accessory(name) @@ -42,7 +41,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base docker :ps, *service_filter end - def logs(timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil) pipe \ docker(:logs, service_name, (" --since #{since}" if since), (" --tail #{lines}" if lines), ("--timestamps" if timestamps), "2>&1"), @@ -56,7 +54,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base (%(grep "#{grep}"#{" #{grep_options}" if grep_options}) if grep) end - def execute_in_existing_container(*command, interactive: false) docker :exec, ("-it" if interactive), @@ -87,7 +84,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base super command, host: hosts.first end - def ensure_local_file_present(local_file) if !local_file.is_a?(StringIO) && !Pathname.new(local_file).exist? raise "Missing file: #{local_file}" diff --git a/lib/kamal/commands/app/assets.rb b/lib/kamal/commands/app/assets.rb index c1e65d18..21ae4d5f 100644 --- a/lib/kamal/commands/app/assets.rb +++ b/lib/kamal/commands/app/assets.rb @@ -4,10 +4,10 @@ module Kamal::Commands::App::Assets combine \ make_directory(role.asset_extracted_directory), - [ *docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true" ], - docker(:run, "--name", asset_container, "--detach", "--rm", "--entrypoint", "sleep", config.absolute_image, "1000000"), - docker(:cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_directory), - docker(:stop, "-t 1", asset_container), + [ *docker(:container, :rm, asset_container, "2> /dev/null"), "|| true" ], + docker(:container, :create, "--name", asset_container, config.absolute_image), + docker(:container, :cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_directory), + docker(:container, :rm, asset_container), by: "&&" end diff --git a/lib/kamal/commands/app/execution.rb b/lib/kamal/commands/app/execution.rb index 4434c26a..e1289fd8 100644 --- a/lib/kamal/commands/app/execution.rb +++ b/lib/kamal/commands/app/execution.rb @@ -7,13 +7,15 @@ module Kamal::Commands::App::Execution *command end - def execute_in_new_container(*command, interactive: false, env:) + def execute_in_new_container(*command, interactive: false, detach: false, env:) docker :run, ("-it" if interactive), - "--rm", + ("--detach" if detach), + ("--rm" unless detach), "--network", "kamal", *role&.env_args(host), *argumentize("--env", env), + *role.logging_args, *config.volume_args, *role&.option_args, config.absolute_image, diff --git a/lib/kamal/commands/app/logging.rb b/lib/kamal/commands/app/logging.rb index ad66f370..d5c8d5c7 100644 --- a/lib/kamal/commands/app/logging.rb +++ b/lib/kamal/commands/app/logging.rb @@ -1,18 +1,28 @@ module Kamal::Commands::App::Logging - def logs(version: nil, timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil) + def logs(container_id: nil, timestamps: true, since: nil, lines: nil, grep: nil, grep_options: nil) pipe \ - version ? container_id_for_version(version) : current_running_container_id, + container_id_command(container_id), "xargs docker logs#{" --timestamps" if timestamps}#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1", ("grep '#{grep}'#{" #{grep_options}" if grep_options}" if grep) end - def follow_logs(host:, timestamps: true, lines: nil, grep: nil, grep_options: nil) + def follow_logs(host:, container_id: nil, timestamps: true, lines: nil, grep: nil, grep_options: nil) run_over_ssh \ pipe( - current_running_container_id, + container_id_command(container_id), "xargs docker logs#{" --timestamps" if timestamps}#{" --tail #{lines}" if lines} --follow 2>&1", (%(grep "#{grep}"#{" #{grep_options}" if grep_options}) if grep) ), host: host end + + private + + def container_id_command(container_id) + case container_id + when Array then container_id + when String, Symbol then "echo #{container_id}" + else current_running_container_id + end + end end diff --git a/lib/kamal/commands/base.rb b/lib/kamal/commands/base.rb index e71470e4..a99e1563 100644 --- a/lib/kamal/commands/base.rb +++ b/lib/kamal/commands/base.rb @@ -11,7 +11,7 @@ module Kamal::Commands end def run_over_ssh(*command, host:) - "ssh#{ssh_proxy_args} -t #{config.ssh.user}@#{host} -p #{config.ssh.port} '#{command.join(" ").gsub("'", "'\\\\''")}'" + "ssh#{ssh_proxy_args}#{ssh_keys_args} -t #{config.ssh.user}@#{host} -p #{config.ssh.port} '#{command.join(" ").gsub("'", "'\\\\''")}'" end def container_id_for(container_name:, only_running: false) @@ -34,6 +34,12 @@ module Kamal::Commands [ :rm, path ] end + def ensure_docker_installed + combine \ + ensure_local_docker_installed, + ensure_local_buildx_installed + end + private def combine(*commands, by: "&&") commands @@ -98,5 +104,23 @@ module Kamal::Commands " -o ProxyCommand='#{config.ssh.proxy.command_line_template}'" end end + + def ssh_keys_args + "#{ ssh_keys.join("") if ssh_keys}" + "#{" -o IdentitiesOnly=yes" if config.ssh&.keys_only}" + end + + def ssh_keys + config.ssh.keys&.map do |key| + " -i #{key}" + end + end + + def ensure_local_docker_installed + docker "--version" + end + + def ensure_local_buildx_installed + docker :buildx, "version" + end end end diff --git a/lib/kamal/commands/builder.rb b/lib/kamal/commands/builder.rb index c50bc3c5..c9408ad9 100644 --- a/lib/kamal/commands/builder.rb +++ b/lib/kamal/commands/builder.rb @@ -2,7 +2,7 @@ require "active_support/core_ext/string/filters" class Kamal::Commands::Builder < Kamal::Commands::Base delegate :create, :remove, :push, :clean, :pull, :info, :inspect_builder, :validate_image, :first_mirror, to: :target - delegate :local?, :remote?, :pack?, to: "config.builder" + delegate :local?, :remote?, :pack?, :cloud?, to: "config.builder" include Clone @@ -19,6 +19,8 @@ class Kamal::Commands::Builder < Kamal::Commands::Base end elsif pack? pack + elsif cloud? + cloud else local end @@ -40,22 +42,7 @@ class Kamal::Commands::Builder < Kamal::Commands::Base @pack ||= Kamal::Commands::Builder::Pack.new(config) end - def ensure_local_dependencies_installed - if name.native? - ensure_local_docker_installed - else - combine \ - ensure_local_docker_installed, - ensure_local_buildx_installed - end + def cloud + @cloud ||= Kamal::Commands::Builder::Cloud.new(config) end - - private - def ensure_local_docker_installed - docker "--version" - end - - def ensure_local_buildx_installed - docker :buildx, "version" - end end diff --git a/lib/kamal/commands/builder/cloud.rb b/lib/kamal/commands/builder/cloud.rb new file mode 100644 index 00000000..f7b97c50 --- /dev/null +++ b/lib/kamal/commands/builder/cloud.rb @@ -0,0 +1,22 @@ +class Kamal::Commands::Builder::Cloud < Kamal::Commands::Builder::Base + # Expects `driver` to be of format "cloud docker-org-name/builder-name" + + def create + docker :buildx, :create, "--driver", driver + end + + def remove + docker :buildx, :rm, builder_name + end + + private + def builder_name + driver.gsub(/[ \/]/, "-") + end + + def inspect_buildx + pipe \ + docker(:buildx, :inspect, builder_name), + grep("-q", "Endpoint:.*cloud://.*") + end +end diff --git a/lib/kamal/commands/registry.rb b/lib/kamal/commands/registry.rb index 69f95360..b17fbf31 100644 --- a/lib/kamal/commands/registry.rb +++ b/lib/kamal/commands/registry.rb @@ -1,14 +1,16 @@ class Kamal::Commands::Registry < Kamal::Commands::Base - delegate :registry, to: :config + def login(registry_config: nil) + registry_config ||= config.registry - def login docker :login, - registry.server, - "-u", sensitive(Kamal::Utils.escape_shell_value(registry.username)), - "-p", sensitive(Kamal::Utils.escape_shell_value(registry.password)) + registry_config.server, + "-u", sensitive(Kamal::Utils.escape_shell_value(registry_config.username)), + "-p", sensitive(Kamal::Utils.escape_shell_value(registry_config.password)) end - def logout - docker :logout, registry.server + def logout(registry_config: nil) + registry_config ||= config.registry + + docker :logout, registry_config.server end end diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index 021e5e49..023a2d7c 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -14,7 +14,7 @@ class Kamal::Configuration include Validation - PROXY_MINIMUM_VERSION = "v0.8.2" + PROXY_MINIMUM_VERSION = "v0.8.4" PROXY_HTTP_PORT = 80 PROXY_HTTPS_PORT = 443 PROXY_LOG_MAX_SIZE = "10m" @@ -37,7 +37,7 @@ class Kamal::Configuration if file.exist? # Newer Psych doesn't load aliases by default load_method = YAML.respond_to?(:unsafe_load) ? :unsafe_load : :load - YAML.send(load_method, ERB.new(IO.read(file)).result).symbolize_keys + YAML.send(load_method, ERB.new(File.read(file)).result).symbolize_keys else raise "Configuration file not found in #{file}" end @@ -59,7 +59,7 @@ class Kamal::Configuration # Eager load config to validate it, these are first as they have dependencies later on @servers = Servers.new(config: self) - @registry = Registry.new(config: self) + @registry = Registry.new(config: @raw_config, secrets: secrets) @accessories = @raw_config.accessories&.keys&.collect { |name| Accessory.new(name, config: self) } || [] @aliases = @raw_config.aliases&.keys&.to_h { |name| [ name, Alias.new(name, config: self) ] } || {} @@ -82,7 +82,6 @@ class Kamal::Configuration ensure_unique_hosts_for_ssl_roles end - def version=(version) @declared_version = version end @@ -106,7 +105,6 @@ class Kamal::Configuration raw_config.minimum_version end - def roles servers.roles end @@ -119,7 +117,6 @@ class Kamal::Configuration accessories.detect { |a| a.name == name.to_s } end - def all_hosts (roles + accessories).flat_map(&:hosts).uniq end @@ -180,7 +177,6 @@ class Kamal::Configuration raw_config.retain_containers || 5 end - def volume_args if raw_config.volumes.present? argumentize "--volume", raw_config.volumes @@ -193,7 +189,6 @@ class Kamal::Configuration logging.args end - def readiness_delay raw_config.readiness_delay || 7 end @@ -206,7 +201,6 @@ class Kamal::Configuration raw_config.drain_timeout || 30 end - def run_directory ".kamal" end @@ -227,7 +221,6 @@ class Kamal::Configuration File.join app_directory, "assets" end - def hooks_path raw_config.hooks_path || ".kamal/hooks" end @@ -236,7 +229,6 @@ class Kamal::Configuration raw_config.asset_path end - def env_tags @env_tags ||= if (tags = raw_config.env["tags"]) tags.collect { |name, config| Env::Tag.new(name, config: config, secrets: secrets) } @@ -249,8 +241,16 @@ class Kamal::Configuration env_tags.detect { |t| t.name == name.to_s } end - def proxy_publish_args(http_port, https_port) - argumentize "--publish", [ "#{http_port}:#{PROXY_HTTP_PORT}", "#{https_port}:#{PROXY_HTTPS_PORT}" ] + def proxy_publish_args(http_port, https_port, bind_ips = nil) + ensure_valid_bind_ips(bind_ips) + + (bind_ips || [ nil ]).map do |bind_ip| + bind_ip = format_bind_ip(bind_ip) + publish_http = [ bind_ip, http_port, PROXY_HTTP_PORT ].compact.join(":") + publish_https = [ bind_ip, https_port, PROXY_HTTPS_PORT ].compact.join(":") + + argumentize "--publish", [ publish_http, publish_https ] + end.join(" ") end def proxy_logging_args(max_size) @@ -277,7 +277,6 @@ class Kamal::Configuration File.join proxy_directory, "options" end - def to_h { roles: role_names, @@ -344,6 +343,15 @@ class Kamal::Configuration true end + def ensure_valid_bind_ips(bind_ips) + bind_ips.present? && bind_ips.each do |ip| + next if ip =~ Resolv::IPv4::Regex || ip =~ Resolv::IPv6::Regex + raise ArgumentError, "Invalid publish IP address: #{ip}" + end + + true + end + def ensure_retain_containers_valid raise Kamal::ConfigurationError, "Must retain at least 1 container" if retain_containers < 1 @@ -375,6 +383,15 @@ class Kamal::Configuration true end + def format_bind_ip(ip) + # Ensure IPv6 address inside square brackets - e.g. [::1] + if ip =~ Resolv::IPv6::Regex && ip !~ /\[.*\]/ + "[#{ip}]" + else + ip + end + end + def role_names raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort end diff --git a/lib/kamal/configuration/accessory.rb b/lib/kamal/configuration/accessory.rb index 2728607d..ccb845fd 100644 --- a/lib/kamal/configuration/accessory.rb +++ b/lib/kamal/configuration/accessory.rb @@ -5,7 +5,7 @@ class Kamal::Configuration::Accessory delegate :argumentize, :optionize, to: Kamal::Utils - attr_reader :name, :accessory_config, :env, :proxy + attr_reader :name, :env, :proxy, :registry def initialize(name, config:) @name, @config, @accessory_config = name.inquiry, config, config.raw_config["accessories"][name] @@ -16,12 +16,9 @@ class Kamal::Configuration::Accessory context: "accessories/#{name}", with: Kamal::Configuration::Validator::Accessory - @env = Kamal::Configuration::Env.new \ - config: accessory_config.fetch("env", {}), - secrets: config.secrets, - context: "accessories/#{name}/env" - - initialize_proxy if running_proxy? + @env = initialize_env + @proxy = initialize_proxy if running_proxy? + @registry = initialize_registry if accessory_config["registry"].present? end def service_name @@ -29,7 +26,7 @@ class Kamal::Configuration::Accessory end def image - accessory_config["image"] + [ registry&.server, accessory_config["image"] ].compact.join("/") end def hosts @@ -109,18 +106,32 @@ class Kamal::Configuration::Accessory end def running_proxy? - @accessory_config["proxy"].present? - end - - def initialize_proxy - @proxy = Kamal::Configuration::Proxy.new \ - config: config, - proxy_config: accessory_config["proxy"], - context: "accessories/#{name}/proxy" + accessory_config["proxy"].present? end private - attr_accessor :config + attr_reader :config, :accessory_config + + def initialize_env + Kamal::Configuration::Env.new \ + config: accessory_config.fetch("env", {}), + secrets: config.secrets, + context: "accessories/#{name}/env" + end + + def initialize_proxy + Kamal::Configuration::Proxy.new \ + config: config, + proxy_config: accessory_config["proxy"], + context: "accessories/#{name}/proxy" + end + + def initialize_registry + Kamal::Configuration::Registry.new \ + config: accessory_config, + secrets: config.secrets, + context: "accessories/#{name}/registry" + end def default_labels { "service" => service_name } @@ -142,7 +153,7 @@ class Kamal::Configuration::Accessory end def read_dynamic_file(local_file) - StringIO.new(ERB.new(IO.read(local_file)).result) + StringIO.new(ERB.new(File.read(local_file)).result) end def expand_remote_file(remote_file) diff --git a/lib/kamal/configuration/builder.rb b/lib/kamal/configuration/builder.rb index bb8d6058..6a7deecc 100644 --- a/lib/kamal/configuration/builder.rb +++ b/lib/kamal/configuration/builder.rb @@ -53,6 +53,10 @@ class Kamal::Configuration::Builder !local_disabled? && (arches.empty? || local_arches.any?) end + def cloud? + driver.start_with? "cloud" + end + def cached? !!builder_config["cache"] end diff --git a/lib/kamal/configuration/docs/accessory.yml b/lib/kamal/configuration/docs/accessory.yml index fab2989f..571d7217 100644 --- a/lib/kamal/configuration/docs/accessory.yml +++ b/lib/kamal/configuration/docs/accessory.yml @@ -23,9 +23,27 @@ accessories: # Image # - # The Docker image to use, prefix it with a registry if not using Docker Hub: + # The Docker image to use. + # Prefix it with its server when using root level registry different from Docker Hub. + # Define registry directly or via anchors when it differs from root level registry. image: mysql:8.0 + # Registry + # + # By default accessories use Docker Hub registry. + # You can specify different registry per accessory with this option. + # Don't prefix image with this registry server. + # Use anchors if you need to set the same specific registry for several accessories. + # + # ```yml + # registry: + # <<: *specific-registry + # ``` + # + # See kamal docs registry for more information: + registry: + ... + # Accessory hosts # # Specify one of `host`, `hosts`, or `roles`: @@ -43,8 +61,8 @@ accessories: # Port mappings # - # See https://docs.docker.com/network/, and especially note the warning about the security - # implications of exposing ports publicly. + # See [https://docs.docker.com/network/](https://docs.docker.com/network/), and + # especially note the warning about the security implications of exposing ports publicly. port: "127.0.0.1:3306:3306" # Labels @@ -101,4 +119,4 @@ accessories: # Proxy # proxy: - ... \ No newline at end of file + ... diff --git a/lib/kamal/configuration/docs/alias.yml b/lib/kamal/configuration/docs/alias.yml index 32c37bad..6c46b695 100644 --- a/lib/kamal/configuration/docs/alias.yml +++ b/lib/kamal/configuration/docs/alias.yml @@ -5,12 +5,12 @@ # For example, for a Rails app, you might open a console with: # # ```shell -# kamal app exec -i -r console "rails console" +# kamal app exec -i --reuse "bin/rails console" # ``` # # By defining an alias, like this: aliases: - console: app exec -r console -i "rails console" + console: app exec -i --reuse "bin/rails console" # You can now open the console with: # # ```shell diff --git a/lib/kamal/configuration/docs/builder.yml b/lib/kamal/configuration/docs/builder.yml index 6549937d..22c310f0 100644 --- a/lib/kamal/configuration/docs/builder.yml +++ b/lib/kamal/configuration/docs/builder.yml @@ -114,6 +114,9 @@ builder: # # The build driver to use, defaults to `docker-container`: driver: docker + # + # If you want to use Docker Build Cloud (https://www.docker.com/products/build-cloud/), you can set the driver to: + driver: cloud org-name/builder-name # Provenance # diff --git a/lib/kamal/configuration/docs/proxy.yml b/lib/kamal/configuration/docs/proxy.yml index 76ec3e41..9ed6a97f 100644 --- a/lib/kamal/configuration/docs/proxy.yml +++ b/lib/kamal/configuration/docs/proxy.yml @@ -46,9 +46,22 @@ proxy: # The host value must point to the server we are deploying to, and port 443 must be # open for the Let's Encrypt challenge to succeed. # + # If you set `ssl` to `true`, `kamal-proxy` will stop forwarding headers to your app, + # unless you explicitly set `forward_headers: true` + # # Defaults to `false`: ssl: true + # Forward headers + # + # Whether to forward the `X-Forwarded-For` and `X-Forwarded-Proto` headers. + # + # If you are behind a trusted proxy, you can set this to `true` to forward the headers. + # + # By default, kamal-proxy will not forward the headers if the `ssl` option is set to `true`, and + # will forward them if it is set to `false`. + forward_headers: true + # Response timeout # # How long to wait for requests to complete before timing out, defaults to 30 seconds: @@ -93,13 +106,3 @@ proxy: response_headers: - X-Request-ID - X-Request-Start - - # Forward headers - # - # Whether to forward the `X-Forwarded-For` and `X-Forwarded-Proto` headers. - # - # If you are behind a trusted proxy, you can set this to `true` to forward the headers. - # - # By default, kamal-proxy will not forward the headers if the `ssl` option is set to `true`, and - # will forward them if it is set to `false`. - forward_headers: true diff --git a/lib/kamal/configuration/docs/registry.yml b/lib/kamal/configuration/docs/registry.yml index 84f9c7c1..4411fd4d 100644 --- a/lib/kamal/configuration/docs/registry.yml +++ b/lib/kamal/configuration/docs/registry.yml @@ -2,6 +2,10 @@ # # The default registry is Docker Hub, but you can change it using `registry/server`. # +# By default, Docker Hub creates public repositories. To avoid making your images public, +# set up a private repository before deploying, or change the default repository privacy +# settings to private in your [Docker Hub settings](https://hub.docker.com/repository-settings/default-privacy). +# # A reference to a secret (in this case, `DOCKER_REGISTRY_TOKEN`) will look up the secret # in the local environment: registry: diff --git a/lib/kamal/configuration/registry.rb b/lib/kamal/configuration/registry.rb index 763cf976..d3fba515 100644 --- a/lib/kamal/configuration/registry.rb +++ b/lib/kamal/configuration/registry.rb @@ -1,12 +1,10 @@ class Kamal::Configuration::Registry include Kamal::Configuration::Validation - attr_reader :registry_config, :secrets - - def initialize(config:) - @registry_config = config.raw_config.registry || {} - @secrets = config.secrets - validate! registry_config, with: Kamal::Configuration::Validator::Registry + def initialize(config:, secrets:, context: "registry") + @registry_config = config["registry"] || {} + @secrets = secrets + validate! registry_config, context: context, with: Kamal::Configuration::Validator::Registry end def server @@ -22,6 +20,8 @@ class Kamal::Configuration::Registry end private + attr_reader :registry_config, :secrets + def lookup(key) if registry_config[key].is_a?(Array) secrets[registry_config[key].first] diff --git a/lib/kamal/configuration/role.rb b/lib/kamal/configuration/role.rb index 708e77fc..c6bd8783 100644 --- a/lib/kamal/configuration/role.rb +++ b/lib/kamal/configuration/role.rb @@ -10,7 +10,7 @@ class Kamal::Configuration::Role def initialize(name, config:) @name, @config = name.inquiry, config validate! \ - specializations, + role_config, example: validation_yml["servers"]["workers"], context: "servers/#{name}", with: Kamal::Configuration::Validator::Role @@ -204,11 +204,11 @@ class Kamal::Configuration::Role end def specializations - if config.raw_config.servers.is_a?(Array) || config.raw_config.servers[name].is_a?(Array) - {} - else - config.raw_config.servers[name] - end + @specializations ||= role_config.is_a?(Array) ? {} : role_config + end + + def role_config + @role_config ||= config.raw_config.servers.is_a?(Array) ? {} : config.raw_config.servers[name] end def custom_labels diff --git a/lib/kamal/configuration/validator/role.rb b/lib/kamal/configuration/validator/role.rb index ce28c039..de7a1969 100644 --- a/lib/kamal/configuration/validator/role.rb +++ b/lib/kamal/configuration/validator/role.rb @@ -3,7 +3,7 @@ class Kamal::Configuration::Validator::Role < Kamal::Configuration::Validator validate_type! config, Array, Hash if config.is_a?(Array) - validate_servers! "servers", config + validate_servers!(config) else super end diff --git a/lib/kamal/secrets.rb b/lib/kamal/secrets.rb index 7a382f98..9d4b0ff7 100644 --- a/lib/kamal/secrets.rb +++ b/lib/kamal/secrets.rb @@ -32,7 +32,7 @@ class Kamal::Secrets private def secrets @secrets ||= secrets_files.inject({}) do |secrets, secrets_file| - secrets.merge!(::Dotenv.parse(secrets_file)) + secrets.merge!(::Dotenv.parse(secrets_file, overwrite: true)) end end diff --git a/lib/kamal/secrets/adapters.rb b/lib/kamal/secrets/adapters.rb index 439c7208..70dfb2b2 100644 --- a/lib/kamal/secrets/adapters.rb +++ b/lib/kamal/secrets/adapters.rb @@ -3,6 +3,8 @@ module Kamal::Secrets::Adapters def self.lookup(name) name = "one_password" if name.downcase == "1password" name = "last_pass" if name.downcase == "lastpass" + name = "gcp_secret_manager" if name.downcase == "gcp" + name = "bitwarden_secrets_manager" if name.downcase == "bitwarden-sm" adapter_class(name) end diff --git a/lib/kamal/secrets/adapters/aws_secrets_manager.rb b/lib/kamal/secrets/adapters/aws_secrets_manager.rb index e23ea1f1..48add1ac 100644 --- a/lib/kamal/secrets/adapters/aws_secrets_manager.rb +++ b/lib/kamal/secrets/adapters/aws_secrets_manager.rb @@ -4,22 +4,30 @@ class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Ba nil end - def fetch_secrets(secrets, account:, session:) + def fetch_secrets(secrets, from:, account:, session:) {}.tap do |results| - JSON.parse(get_from_secrets_manager(secrets, account: account))["SecretValues"].each do |secret| + get_from_secrets_manager(prefixed_secrets(secrets, from: from), account: account).each do |secret| secret_name = secret["Name"] secret_string = JSON.parse(secret["SecretString"]) secret_string.each do |key, value| results["#{secret_name}/#{key}"] = value end + rescue JSON::ParserError + results["#{secret_name}"] = secret["SecretString"] end end end def get_from_secrets_manager(secrets, account:) - `aws secretsmanager batch-get-secret-value --secret-id-list #{secrets.map(&:shellescape).join(" ")} --profile #{account.shellescape}`.tap do - raise RuntimeError, "Could not read #{secret} from AWS Secrets Manager" unless $?.success? + `aws secretsmanager batch-get-secret-value --secret-id-list #{secrets.map(&:shellescape).join(" ")} --profile #{account.shellescape}`.tap do |secrets| + raise RuntimeError, "Could not read #{secrets} from AWS Secrets Manager" unless $?.success? + + secrets = JSON.parse(secrets) + + return secrets["SecretValues"] unless secrets["Errors"].present? + + raise RuntimeError, secrets["Errors"].map { |error| "#{error['SecretId']}: #{error['Message']}" }.join(" ") end end diff --git a/lib/kamal/secrets/adapters/base.rb b/lib/kamal/secrets/adapters/base.rb index fc66bb34..c74f7c41 100644 --- a/lib/kamal/secrets/adapters/base.rb +++ b/lib/kamal/secrets/adapters/base.rb @@ -7,8 +7,7 @@ class Kamal::Secrets::Adapters::Base check_dependencies! session = login(account) - full_secrets = secrets.map { |secret| [ from, secret ].compact.join("/") } - fetch_secrets(full_secrets, account: account, session: session) + fetch_secrets(secrets, from: from, account: account, session: session) end def requires_account? @@ -27,4 +26,8 @@ class Kamal::Secrets::Adapters::Base def check_dependencies! raise NotImplementedError end + + def prefixed_secrets(secrets, from:) + secrets.map { |secret| [ from, secret ].compact.join("/") } + end end diff --git a/lib/kamal/secrets/adapters/bitwarden.rb b/lib/kamal/secrets/adapters/bitwarden.rb index 5dfb72db..6bd4fb25 100644 --- a/lib/kamal/secrets/adapters/bitwarden.rb +++ b/lib/kamal/secrets/adapters/bitwarden.rb @@ -21,9 +21,9 @@ class Kamal::Secrets::Adapters::Bitwarden < Kamal::Secrets::Adapters::Base session end - def fetch_secrets(secrets, account:, session:) + def fetch_secrets(secrets, from:, account:, session:) {}.tap do |results| - items_fields(secrets).each do |item, fields| + items_fields(prefixed_secrets(secrets, from: from)).each do |item, fields| item_json = run_command("get item #{item.shellescape}", session: session, raw: true) raise RuntimeError, "Could not read #{item} from Bitwarden" unless $?.success? item_json = JSON.parse(item_json) diff --git a/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb b/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb new file mode 100644 index 00000000..66afbe70 --- /dev/null +++ b/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb @@ -0,0 +1,72 @@ +class Kamal::Secrets::Adapters::BitwardenSecretsManager < Kamal::Secrets::Adapters::Base + def requires_account? + false + end + + private + LIST_ALL_SELECTOR = "all" + LIST_ALL_FROM_PROJECT_SUFFIX = "/all" + LIST_COMMAND = "secret list -o env" + GET_COMMAND = "secret get -o env" + + def fetch_secrets(secrets, from:, account:, session:) + raise RuntimeError, "You must specify what to retrieve from Bitwarden Secrets Manager" if secrets.length == 0 + + secrets = prefixed_secrets(secrets, from: from) + command, project = extract_command_and_project(secrets) + + {}.tap do |results| + if command.nil? + secrets.each do |secret_uuid| + secret = run_command("#{GET_COMMAND} #{secret_uuid.shellescape}") + raise RuntimeError, "Could not read #{secret_uuid} from Bitwarden Secrets Manager" unless $?.success? + key, value = parse_secret(secret) + results[key] = value + end + else + secrets = run_command(command) + raise RuntimeError, "Could not read secrets from Bitwarden Secrets Manager" unless $?.success? + secrets.split("\n").each do |secret| + key, value = parse_secret(secret) + results[key] = value + end + end + end + end + + def extract_command_and_project(secrets) + if secrets.length == 1 + if secrets[0] == LIST_ALL_SELECTOR + [ LIST_COMMAND, nil ] + elsif secrets[0].end_with?(LIST_ALL_FROM_PROJECT_SUFFIX) + project = secrets[0].split(LIST_ALL_FROM_PROJECT_SUFFIX).first + [ "#{LIST_COMMAND} #{project.shellescape}", project ] + end + end + end + + def parse_secret(secret) + key, value = secret.split("=", 2) + value = value.gsub(/^"|"$/, "") + [ key, value ] + end + + def run_command(command, session: nil) + full_command = [ "bws", command ].join(" ") + `#{full_command}` + end + + def login(account) + run_command("run 'echo OK'") + raise RuntimeError, "Could not authenticate to Bitwarden Secrets Manager. Did you set a valid access token?" unless $?.success? + end + + def check_dependencies! + raise RuntimeError, "Bitwarden Secrets Manager CLI is not installed" unless cli_installed? + end + + def cli_installed? + `bws --version 2> /dev/null` + $?.success? + end +end diff --git a/lib/kamal/secrets/adapters/doppler.rb b/lib/kamal/secrets/adapters/doppler.rb index 64d644f7..90b2c63b 100644 --- a/lib/kamal/secrets/adapters/doppler.rb +++ b/lib/kamal/secrets/adapters/doppler.rb @@ -16,8 +16,21 @@ class Kamal::Secrets::Adapters::Doppler < Kamal::Secrets::Adapters::Base $?.success? end - def fetch_secrets(secrets, **) - project_and_config_flags = "" + def fetch_secrets(secrets, from:, **) + secrets = prefixed_secrets(secrets, from: from) + flags = secrets_get_flags(secrets) + + secret_names = secrets.collect { |s| s.split("/").last } + + items = `doppler secrets get #{secret_names.map(&:shellescape).join(" ")} --json #{flags}` + raise RuntimeError, "Could not read #{secrets} from Doppler" unless $?.success? + + items = JSON.parse(items) + + items.transform_values { |value| value["computed"] } + end + + def secrets_get_flags(secrets) unless service_token_set? project, config, _ = secrets.first.split("/") @@ -27,15 +40,6 @@ class Kamal::Secrets::Adapters::Doppler < Kamal::Secrets::Adapters::Base project_and_config_flags = "-p #{project.shellescape} -c #{config.shellescape}" end - - secret_names = secrets.collect { |s| s.split("/").last } - - items = `doppler secrets get #{secret_names.map(&:shellescape).join(" ")} --json #{project_and_config_flags}` - raise RuntimeError, "Could not read #{secrets} from Doppler" unless $?.success? - - items = JSON.parse(items) - - items.transform_values { |value| value["computed"] } end def service_token_set? diff --git a/lib/kamal/secrets/adapters/enpass.rb b/lib/kamal/secrets/adapters/enpass.rb new file mode 100644 index 00000000..96dea11a --- /dev/null +++ b/lib/kamal/secrets/adapters/enpass.rb @@ -0,0 +1,71 @@ +## +# Enpass is different from most password managers, in a way that it's offline and doesn't need an account. +# +# Usage +# +# Fetch all password from FooBar item +# `kamal secrets fetch --adapter enpass --from /Users/YOUR_USERNAME/Library/Containers/in.sinew.Enpass-Desktop/Data/Documents/Vaults/primary FooBar` +# +# Fetch only DB_PASSWORD from FooBar item +# `kamal secrets fetch --adapter enpass --from /Users/YOUR_USERNAME/Library/Containers/in.sinew.Enpass-Desktop/Data/Documents/Vaults/primary FooBar/DB_PASSWORD` +class Kamal::Secrets::Adapters::Enpass < Kamal::Secrets::Adapters::Base + def requires_account? + false + end + + private + def fetch_secrets(secrets, from:, account:, session:) + secrets_titles = fetch_secret_titles(secrets) + + result = `enpass-cli -json -vault #{from.shellescape} show #{secrets_titles.map(&:shellescape).join(" ")}`.strip + + parse_result_and_take_secrets(result, secrets) + end + + def check_dependencies! + raise RuntimeError, "Enpass CLI is not installed" unless cli_installed? + end + + def cli_installed? + `enpass-cli version 2> /dev/null` + $?.success? + end + + def login(account) + nil + end + + def fetch_secret_titles(secrets) + secrets.reduce(Set.new) do |secret_titles, secret| + # Sometimes secrets contain a '/', when the intent is to fetch a single password for an item. Example: FooBar/DB_PASSWORD + # Another case is, when the intent is to fetch all passwords for an item. Example: FooBar (and FooBar may have multiple different passwords) + key, separator, value = secret.rpartition("/") + if key.empty? + secret_titles << value + else + secret_titles << key + end + end.to_a + end + + def parse_result_and_take_secrets(unparsed_result, secrets) + result = JSON.parse(unparsed_result) + + result.reduce({}) do |secrets_with_passwords, item| + title = item["title"] + label = item["label"] + password = item["password"] + + if title && password.present? + key = [ title, label ].compact.reject(&:empty?).join("/") + + if secrets.include?(title) || secrets.include?(key) + raise RuntimeError, "#{key} is present more than once" if secrets_with_passwords[key] + secrets_with_passwords[key] = password + end + end + + secrets_with_passwords + end + end +end diff --git a/lib/kamal/secrets/adapters/gcp_secret_manager.rb b/lib/kamal/secrets/adapters/gcp_secret_manager.rb new file mode 100644 index 00000000..8ce381ff --- /dev/null +++ b/lib/kamal/secrets/adapters/gcp_secret_manager.rb @@ -0,0 +1,112 @@ +class Kamal::Secrets::Adapters::GcpSecretManager < Kamal::Secrets::Adapters::Base + private + def login(account) + # Since only the account option is passed from the cli, we'll use it for both account and service account + # impersonation. + # + # Syntax: + # ACCOUNT: USER | USER "|" DELEGATION_CHAIN + # USER: DEFAULT_USER | EMAIL + # DELEGATION_CHAIN: EMAIL | EMAIL "," DELEGATION_CHAIN + # EMAIL: + # DEFAULT_USER: "default" + # + # Some valid examples: + # - "my-user@example.com" sets the user + # - "my-user@example.com|my-service-user@example.com" will use my-user and enable service account impersonation as my-service-user + # - "default" will use the default user and no impersonation + # - "default|my-service-user@example.com" will use the default user, and enable service account impersonation as my-service-user + # - "default|my-service-user@example.com,another-service-user@example.com" same as above, but with an impersonation delegation chain + + unless logged_in? + `gcloud auth login` + raise RuntimeError, "could not login to gcloud" unless logged_in? + end + + nil + end + + def fetch_secrets(secrets, from:, account:, session:) + user, service_account = parse_account(account) + + {}.tap do |results| + secrets_with_metadata(prefixed_secrets(secrets, from: from)).each do |secret, (project, secret_name, secret_version)| + item_name = "#{project}/#{secret_name}" + results[item_name] = fetch_secret(project, secret_name, secret_version, user, service_account) + raise RuntimeError, "Could not read #{item_name} from Google Secret Manager" unless $?.success? + end + end + end + + def fetch_secret(project, secret_name, secret_version, user, service_account) + secret = run_command( + "secrets versions access #{secret_version.shellescape} --secret=#{secret_name.shellescape}", + project: project, + user: user, + service_account: service_account + ) + Base64.decode64(secret.dig("payload", "data")) + end + + # The secret needs to at least contain a secret name, but project name, and secret version can also be specified. + # + # The string "default" can be used to refer to the default project configured for gcloud. + # + # The version can be either the string "latest", or a version number. + # + # The following formats are valid: + # + # - The following are all equivalent, and sets project: default, secret name: my-secret, version: latest + # - "my-secret" + # - "default/my-secret" + # - "default/my-secret/latest" + # - "my-secret/latest" in combination with --from=default + # - "my-secret/123" (only in combination with --from=some-project) -> project: some-project, secret name: my-secret, version: 123 + # - "some-project/my-secret/123" -> project: some-project, secret name: my-secret, version: 123 + def secrets_with_metadata(secrets) + {}.tap do |items| + secrets.each do |secret| + parts = secret.split("/") + parts.unshift("default") if parts.length == 1 + project = parts.shift + secret_name = parts.shift + secret_version = parts.shift || "latest" + + items[secret] = [ project, secret_name, secret_version ] + end + end + end + + def run_command(command, project: "default", user: "default", service_account: nil) + full_command = [ "gcloud", command ] + full_command << "--project=#{project.shellescape}" unless project == "default" + full_command << "--account=#{user.shellescape}" unless user == "default" + full_command << "--impersonate-service-account=#{service_account.shellescape}" if service_account + full_command << "--format=json" + full_command = full_command.join(" ") + + result = `#{full_command}`.strip + JSON.parse(result) + end + + def check_dependencies! + raise RuntimeError, "gcloud CLI is not installed" unless cli_installed? + end + + def cli_installed? + `gcloud --version 2> /dev/null` + $?.success? + end + + def logged_in? + JSON.parse(`gcloud auth list --format=json`).any? + end + + def parse_account(account) + account.split("|", 2) + end + + def is_user?(candidate) + candidate.include?("@") + end +end diff --git a/lib/kamal/secrets/adapters/last_pass.rb b/lib/kamal/secrets/adapters/last_pass.rb index 2f95148b..dff872a8 100644 --- a/lib/kamal/secrets/adapters/last_pass.rb +++ b/lib/kamal/secrets/adapters/last_pass.rb @@ -11,7 +11,8 @@ class Kamal::Secrets::Adapters::LastPass < Kamal::Secrets::Adapters::Base `lpass status --color never`.strip == "Logged in as #{account}." end - def fetch_secrets(secrets, account:, session:) + def fetch_secrets(secrets, from:, account:, session:) + secrets = prefixed_secrets(secrets, from: from) items = `lpass show #{secrets.map(&:shellescape).join(" ")} --json` raise RuntimeError, "Could not read #{secrets} from LastPass" unless $?.success? @@ -23,7 +24,7 @@ class Kamal::Secrets::Adapters::LastPass < Kamal::Secrets::Adapters::Base end if (missing_items = secrets - results.keys).any? - raise RuntimeError, "Could not find #{missing_items.join(", ")} in LassPass" + raise RuntimeError, "Could not find #{missing_items.join(", ")} in LastPass" end end end diff --git a/lib/kamal/secrets/adapters/one_password.rb b/lib/kamal/secrets/adapters/one_password.rb index c7f9d5ab..fe454342 100644 --- a/lib/kamal/secrets/adapters/one_password.rb +++ b/lib/kamal/secrets/adapters/one_password.rb @@ -15,9 +15,9 @@ class Kamal::Secrets::Adapters::OnePassword < Kamal::Secrets::Adapters::Base $?.success? end - def fetch_secrets(secrets, account:, session:) + def fetch_secrets(secrets, from:, account:, session:) {}.tap do |results| - vaults_items_fields(secrets).map do |vault, items| + vaults_items_fields(prefixed_secrets(secrets, from: from)).map do |vault, items| items.each do |item, fields| fields_json = JSON.parse(op_item_get(vault, item, fields, account: account, session: session)) fields_json = [ fields_json ] if fields.one? diff --git a/lib/kamal/secrets/adapters/test.rb b/lib/kamal/secrets/adapters/test.rb index 82577a76..ac48960b 100644 --- a/lib/kamal/secrets/adapters/test.rb +++ b/lib/kamal/secrets/adapters/test.rb @@ -4,8 +4,8 @@ class Kamal::Secrets::Adapters::Test < Kamal::Secrets::Adapters::Base true end - def fetch_secrets(secrets, account:, session:) - secrets.to_h { |secret| [ secret, secret.reverse ] } + def fetch_secrets(secrets, from:, account:, session:) + prefixed_secrets(secrets, from: from).to_h { |secret| [ secret, secret.reverse ] } end def check_dependencies! diff --git a/lib/kamal/secrets/adapters/test_optional_account.rb b/lib/kamal/secrets/adapters/test_optional_account.rb deleted file mode 100644 index 3a252e68..00000000 --- a/lib/kamal/secrets/adapters/test_optional_account.rb +++ /dev/null @@ -1,5 +0,0 @@ -class Kamal::Secrets::Adapters::TestOptionalAccount < Kamal::Secrets::Adapters::Test - def requires_account? - false - end -end diff --git a/lib/kamal/version.rb b/lib/kamal/version.rb index 2f9a86b6..9f3a9126 100644 --- a/lib/kamal/version.rb +++ b/lib/kamal/version.rb @@ -1,3 +1,3 @@ module Kamal - VERSION = "2.3.0" + VERSION = "2.4.0" end diff --git a/test/cli/accessory_test.rb b/test/cli/accessory_test.rb index 05431fcb..cc517e59 100644 --- a/test/cli/accessory_test.rb +++ b/test/cli/accessory_test.rb @@ -14,8 +14,8 @@ class CliAccessoryTest < CliTestCase Kamal::Cli::Accessory.any_instance.expects(:upload).with("mysql") run_command("boot", "mysql").tap do |output| - assert_match /docker login.*on 1.1.1.3/, output - assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" private.registry/mysql:5.7 on 1.1.1.3", output end end @@ -24,17 +24,21 @@ class CliAccessoryTest < CliTestCase Kamal::Cli::Accessory.any_instance.expects(:upload).with("mysql") Kamal::Cli::Accessory.any_instance.expects(:directories).with("redis") Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis") + Kamal::Cli::Accessory.any_instance.expects(:directories).with("busybox") + Kamal::Cli::Accessory.any_instance.expects(:upload).with("busybox") run_command("boot", "all").tap do |output| - assert_match /docker login.*on 1.1.1.3/, output - assert_match /docker login.*on 1.1.1.1/, output - assert_match /docker login.*on 1.1.1.2/, output + assert_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.3", output + assert_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.1", output + assert_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.2", output + assert_match "docker login other.registry -u [REDACTED] -p [REDACTED] on 1.1.1.3", output assert_match /docker network create kamal.*on 1.1.1.1/, output assert_match /docker network create kamal.*on 1.1.1.2/, output assert_match /docker network create kamal.*on 1.1.1.3/, output - assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" private.registry/mysql:5.7 on 1.1.1.3", output assert_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output assert_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output + assert_match "docker run --name custom-box --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --env-file .kamal/apps/app/env/accessories/busybox.env --label service=\"custom-box\" other.registry/busybox:latest on 1.1.1.3", output end end @@ -60,13 +64,16 @@ class CliAccessoryTest < CliTestCase end test "reboot all" do - Kamal::Commands::Registry.any_instance.expects(:login).times(3) + Kamal::Commands::Registry.any_instance.expects(:login).times(4) Kamal::Cli::Accessory.any_instance.expects(:stop).with("mysql") Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("mysql") Kamal::Cli::Accessory.any_instance.expects(:boot).with("mysql", prepare: false) Kamal::Cli::Accessory.any_instance.expects(:stop).with("redis") Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("redis") Kamal::Cli::Accessory.any_instance.expects(:boot).with("redis", prepare: false) + Kamal::Cli::Accessory.any_instance.expects(:stop).with("busybox") + Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("busybox") + Kamal::Cli::Accessory.any_instance.expects(:boot).with("busybox", prepare: false) run_command("reboot", "all") end @@ -94,7 +101,7 @@ class CliAccessoryTest < CliTestCase end test "details with non-existent accessory" do - assert_equal "No accessory by the name of 'hello' (options: mysql and redis)", stderred { run_command("details", "hello") } + assert_equal "No accessory by the name of 'hello' (options: mysql, redis, and busybox)", stderred { run_command("details", "hello") } end test "details with all" do @@ -180,6 +187,10 @@ class CliAccessoryTest < CliTestCase Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("redis") Kamal::Cli::Accessory.any_instance.expects(:remove_image).with("redis") Kamal::Cli::Accessory.any_instance.expects(:remove_service_directory).with("redis") + Kamal::Cli::Accessory.any_instance.expects(:stop).with("busybox") + Kamal::Cli::Accessory.any_instance.expects(:remove_container).with("busybox") + Kamal::Cli::Accessory.any_instance.expects(:remove_image).with("busybox") + Kamal::Cli::Accessory.any_instance.expects(:remove_service_directory).with("busybox") run_command("remove", "all", "-y") end @@ -189,7 +200,7 @@ class CliAccessoryTest < CliTestCase end test "remove_image" do - assert_match "docker image rm --force mysql", run_command("remove_image", "mysql") + assert_match "docker image rm --force private.registry/mysql:5.7", run_command("remove_image", "mysql") end test "remove_service_directory" do @@ -201,8 +212,8 @@ class CliAccessoryTest < CliTestCase Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis") run_command("boot", "redis", "--hosts", "1.1.1.1").tap do |output| - assert_match /docker login.*on 1.1.1.1/, output - assert_no_match /docker login.*on 1.1.1.2/, output + assert_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.1", output + assert_no_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.2", output assert_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output assert_no_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.2", output end @@ -213,8 +224,8 @@ class CliAccessoryTest < CliTestCase Kamal::Cli::Accessory.any_instance.expects(:upload).with("redis") run_command("boot", "redis", "--hosts", "1.1.1.1,1.1.1.3").tap do |output| - assert_match /docker login.*on 1.1.1.1/, output - assert_no_match /docker login.*on 1.1.1.3/, output + assert_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.1", output + assert_no_match "docker login private.registry -u [REDACTED] -p [REDACTED] on 1.1.1.3", output assert_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.1", output assert_no_match "docker run --name app-redis --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 6379:6379 --env-file .kamal/apps/app/env/accessories/redis.env --volume $PWD/app-redis/data:/data --label service=\"app-redis\" redis:latest on 1.1.1.3", output end @@ -225,7 +236,7 @@ class CliAccessoryTest < CliTestCase assert_match "Upgrading all accessories on 1.1.1.3,1.1.1.1,1.1.1.2...", output assert_match "docker network create kamal on 1.1.1.3", output assert_match "docker container stop app-mysql on 1.1.1.3", output - assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST="%" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST="%" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" private.registry/mysql:5.7 on 1.1.1.3", output assert_match "Upgraded all accessories on 1.1.1.3,1.1.1.1,1.1.1.2...", output end end @@ -235,14 +246,13 @@ class CliAccessoryTest < CliTestCase assert_match "Upgrading all accessories on 1.1.1.3...", output assert_match "docker network create kamal on 1.1.1.3", output assert_match "docker container stop app-mysql on 1.1.1.3", output - assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST="%" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" mysql:5.7 on 1.1.1.3", output + assert_match "docker run --name app-mysql --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --publish 3306:3306 --env MYSQL_ROOT_HOST="%" --env-file .kamal/apps/app/env/accessories/mysql.env --volume $PWD/app-mysql/etc/mysql/my.cnf:/etc/mysql/my.cnf --volume $PWD/app-mysql/data:/var/lib/mysql --label service=\"app-mysql\" private.registry/mysql:5.7 on 1.1.1.3", output assert_match "Upgraded all accessories on 1.1.1.3", output end end - private def run_command(*command) - stdouted { Kamal::Cli::Accessory.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) } + stdouted { Kamal::Cli::Accessory.start([ *command, "-c", "test/fixtures/deploy_with_accessories_with_different_registries.yml" ]) } end end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 5e76179c..fa5049ba 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -73,7 +73,7 @@ class CliAppTest < CliTestCase run_command("boot", config: :with_assets).tap do |output| assert_match "docker tag dhh/app:latest dhh/app:latest", output assert_match "/usr/bin/env mkdir -p .kamal/apps/app/assets/volumes/web-latest ; cp -rnT .kamal/apps/app/assets/extracted/web-latest .kamal/apps/app/assets/volumes/web-latest ; cp -rnT .kamal/apps/app/assets/extracted/web-latest .kamal/apps/app/assets/volumes/web-123 || true ; cp -rnT .kamal/apps/app/assets/extracted/web-123 .kamal/apps/app/assets/volumes/web-latest || true", output - assert_match "/usr/bin/env mkdir -p .kamal/apps/app/assets/extracted/web-latest && docker stop -t 1 app-web-assets 2> /dev/null || true && docker run --name app-web-assets --detach --rm --entrypoint sleep dhh/app:latest 1000000 && docker cp -L app-web-assets:/public/assets/. .kamal/apps/app/assets/extracted/web-latest && docker stop -t 1 app-web-assets", output + assert_match "/usr/bin/env mkdir -p .kamal/apps/app/assets/extracted/web-latest && docker container rm app-web-assets 2> /dev/null || true && docker container create --name app-web-assets dhh/app:latest && docker container cp -L app-web-assets:/public/assets/. .kamal/apps/app/assets/extracted/web-latest && docker container rm app-web-assets", output assert_match /docker run --detach --restart unless-stopped --name app-web-latest --network kamal --hostname 1.1.1.1-[0-9a-f]{12} /, output assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output assert_match "/usr/bin/env find .kamal/apps/app/assets/extracted -maxdepth 1 -name 'web-*' ! -name web-latest -exec rm -rf \"{}\" + ; find .kamal/apps/app/assets/volumes -maxdepth 1 -name 'web-*' ! -name web-latest -exec rm -rf \"{}\" +", output @@ -263,13 +263,37 @@ class CliAppTest < CliTestCase test "exec" do run_command("exec", "ruby -v").tap do |output| - assert_match "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env dhh/app:latest ruby -v", output + assert_match "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:latest ruby -v", output end end test "exec separate arguments" do run_command("exec", "ruby", " -v").tap do |output| - assert_match "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env dhh/app:latest ruby -v", output + assert_match "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:latest ruby -v", output + end + end + + test "exec detach" do + run_command("exec", "--detach", "ruby -v").tap do |output| + assert_match "docker run --detach --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:latest ruby -v", output + end + end + + test "exec detach with reuse" do + assert_raises(ArgumentError, "Detach is not compatible with reuse") do + run_command("exec", "--detach", "--reuse", "ruby -v") + end + end + + test "exec detach with interactive" do + assert_raises(ArgumentError, "Detach is not compatible with interactive") do + run_command("exec", "--interactive", "--detach", "ruby -v") + end + end + + test "exec detach with interactive and reuse" do + assert_raises(ArgumentError, "Detach is not compatible with interactive or reuse") do + run_command("exec", "--interactive", "--detach", "--reuse", "ruby -v") end end @@ -282,7 +306,7 @@ class CliAppTest < CliTestCase test "exec interactive" do SSHKit::Backend::Abstract.any_instance.expects(:exec) - .with("ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env dhh/app:latest ruby -v'") + .with("ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:latest ruby -v'") run_command("exec", "-i", "ruby -v").tap do |output| assert_match "Get most recent version available as an image...", output assert_match "Launching interactive command with version latest via SSH from new container on 1.1.1.1...", output @@ -329,6 +353,13 @@ class CliAppTest < CliTestCase assert_match "sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow") end + test "logs with follow and container_id" do + SSHKit::Backend::Abstract.any_instance.stubs(:exec) + .with("ssh -t root@1.1.1.1 -p 22 'echo ID123 | xargs docker logs --timestamps --tail 10 --follow 2>&1'") + + assert_match "echo ID123 | xargs docker logs --timestamps --tail 10 --follow 2>&1", run_command("logs", "--follow", "--container-id", "ID123") + end + test "logs with follow and grep" do SSHKit::Backend::Abstract.any_instance.stubs(:exec) .with("ssh -t root@1.1.1.1 -p 22 'sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --follow 2>&1 | grep \"hey\"'") @@ -351,8 +382,10 @@ class CliAppTest < CliTestCase test "version through main" do - stdouted { Kamal::Cli::Main.start([ "app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1" ]) }.tap do |output| - assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output + with_argv([ "app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1" ]) do + stdouted { Kamal::Cli::Main.start }.tap do |output| + assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output + end end end diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb index 4259fa5b..3cc3e2f6 100644 --- a/test/cli/build_test.rb +++ b/test/cli/build_test.rb @@ -155,7 +155,7 @@ class CliBuildTest < CliTestCase .raises(SSHKit::Command::Failed.new("no buildx")) Kamal::Commands::Builder.any_instance.stubs(:native_and_local?).returns(false) - assert_raises(Kamal::Cli::Build::BuildError) { run_command("push") } + assert_raises(Kamal::Cli::DependencyError) { run_command("push") } end test "push pre-build hook failure" do @@ -235,6 +235,12 @@ class CliBuildTest < CliTestCase end end + test "create cloud" do + run_command("create", fixture: :with_cloud_builder).tap do |output| + assert_match /docker buildx create --driver cloud example_org\/cloud_builder/, output + end + end + test "create with error" do stub_setup SSHKit::Backend::Abstract.any_instance.stubs(:execute) @@ -252,6 +258,12 @@ class CliBuildTest < CliTestCase end end + test "remove cloud" do + run_command("remove", fixture: :with_cloud_builder).tap do |output| + assert_match /docker buildx rm cloud-example_org-cloud_builder/, output + end + end + test "details" do SSHKit::Backend::Abstract.any_instance.stubs(:capture) .with(:docker, :context, :ls, "&&", :docker, :buildx, :ls) @@ -274,17 +286,4 @@ class CliBuildTest < CliTestCase SSHKit::Backend::Abstract.any_instance.stubs(:execute) .with { |*args| args[0..1] == [ :docker, :buildx ] } end - - def with_build_directory - build_directory = File.join Dir.tmpdir, "kamal-clones", "app-#{pwd_sha}", "kamal" - FileUtils.mkdir_p build_directory - FileUtils.touch File.join build_directory, "Dockerfile" - yield build_directory + "/" - ensure - FileUtils.rm_rf build_directory - end - - def pwd_sha - Digest::SHA256.hexdigest(Dir.pwd)[0..12] - end end diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb index 27bf7b69..d4b57923 100644 --- a/test/cli/cli_test_case.rb +++ b/test/cli/cli_test_case.rb @@ -51,4 +51,17 @@ class CliTestCase < ActiveSupport::TestCase ensure ARGV.replace(old_argv) end + + def with_build_directory + build_directory = File.join Dir.tmpdir, "kamal-clones", "app-#{pwd_sha}", "kamal" + FileUtils.mkdir_p build_directory + FileUtils.touch File.join build_directory, "Dockerfile" + yield build_directory + "/" + ensure + FileUtils.rm_rf build_directory + end + + def pwd_sha + Digest::SHA256.hexdigest(Dir.pwd)[0..12] + end end diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index cd0efe1f..e901c3ba 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -8,8 +8,7 @@ class CliMainTest < CliTestCase invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:server:bootstrap", [], invoke_options) - Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:accessory:boot", [ "all" ], invoke_options) - Kamal::Cli::Main.any_instance.expects(:deploy) + Kamal::Cli::Main.any_instance.expects(:deploy).with(boot_accessories: true) run_command("setup").tap do |output| assert_match /Ensure Docker is installed.../, output @@ -460,6 +459,7 @@ class CliMainTest < CliTestCase test "run an alias for a console" do run_command("console", config_file: "deploy_with_aliases").tap do |output| + assert_no_match "App Host: 1.1.1.4", output assert_match "docker exec app-console-999 bin/console on 1.1.1.5", output assert_match "App Host: 1.1.1.5", output end @@ -486,6 +486,33 @@ class CliMainTest < CliTestCase end end + test "switch config file with an alias" do + with_config_files do + with_argv([ "other_config" ]) do + stdouted { Kamal::Cli::Main.start }.tap do |output| + assert_match ":service_with_version: app2-999", output + end + end + end + end + + test "switch destination with an alias" do + with_config_files do + with_argv([ "other_destination_config" ]) do + stdouted { Kamal::Cli::Main.start }.tap do |output| + assert_match ":service_with_version: app3-999", output + end + end + end + end + + test "run on primary via alias" do + run_command("primary_details", config_file: "deploy_with_aliases").tap do |output| + assert_match "App Host: 1.1.1.1", output + assert_no_match "App Host: 1.1.1.2", output + end + end + test "upgrade" do invoke_options = { "config_file" => "test/fixtures/deploy_with_accessories.yml", "skip_hooks" => false, "confirmed" => true, "rolling" => false } Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:upgrade", [], invoke_options) @@ -530,6 +557,20 @@ class CliMainTest < CliTestCase end end + def with_config_files + Dir.mktmpdir do |tmpdir| + config_dir = File.join(tmpdir, "config") + FileUtils.mkdir_p(config_dir) + FileUtils.cp "test/fixtures/deploy.yml", config_dir + FileUtils.cp "test/fixtures/deploy2.yml", config_dir + FileUtils.cp "test/fixtures/deploy.elsewhere.yml", config_dir + + Dir.chdir(tmpdir) do + yield + end + end + end + def assert_file(file, content) assert_match content, File.read(file) end diff --git a/test/cli/proxy_test.rb b/test/cli/proxy_test.rb index 0a890451..b9de3e15 100644 --- a/test/cli/proxy_test.rb +++ b/test/cli/proxy_test.rb @@ -55,13 +55,11 @@ class CliProxyTest < CliTestCase run_command("reboot", "-y").tap do |output| assert_match "docker container stop kamal-proxy on 1.1.1.1", output - assert_match "Running docker container stop traefik ; docker container prune --force --filter label=org.opencontainers.image.title=Traefik && docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.1", output assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.1", output assert_match "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image} on 1.1.1.1", output assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target=\"abcdefabcdef:80\" --deploy-timeout=\"6s\" --drain-timeout=\"30s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\" on 1.1.1.1", output assert_match "docker container stop kamal-proxy on 1.1.1.2", output - assert_match "Running docker container stop traefik ; docker container prune --force --filter label=org.opencontainers.image.title=Traefik && docker image prune --all --force --filter label=org.opencontainers.image.title=Traefik on 1.1.1.2", output assert_match "docker container prune --force --filter label=org.opencontainers.image.title=kamal-proxy on 1.1.1.2", output assert_match "docker run --name kamal-proxy --network kamal --detach --restart unless-stopped --volume kamal-proxy-config:/home/kamal-proxy/.config/kamal-proxy $(cat .kamal/proxy/options || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") #{KAMAL.config.proxy_image} on 1.1.1.2", output assert_match "docker exec kamal-proxy kamal-proxy deploy app-web --target=\"abcdefabcdef:80\" --deploy-timeout=\"6s\" --drain-timeout=\"30s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\" on 1.1.1.2", output @@ -281,6 +279,32 @@ class CliProxyTest < CliTestCase end end + test "boot_config set bind IP" do + run_command("boot_config", "set", "--publish-host-ip", "127.0.0.1").tap do |output| + %w[ 1.1.1.1 1.1.1.2 ].each do |host| + assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output + assert_match "Uploading \"--publish 127.0.0.1:80:80 --publish 127.0.0.1:443:443 --log-opt max-size=10m\" to .kamal/proxy/options on #{host}", output + end + end + end + + test "boot_config set multiple bind IPs" do + run_command("boot_config", "set", "--publish-host-ip", "127.0.0.1", "--publish-host-ip", "::1").tap do |output| + %w[ 1.1.1.1 1.1.1.2 ].each do |host| + assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output + assert_match "Uploading \"--publish 127.0.0.1:80:80 --publish 127.0.0.1:443:443 --publish [::1]:80:80 --publish [::1]:443:443 --log-opt max-size=10m\" to .kamal/proxy/options on #{host}", output + end + end + end + + test "boot_config set invalid bind IPs" do + exception = assert_raises do + run_command("boot_config", "set", "--publish-host-ip", "1.2.3.invalidIP", "--publish-host-ip", "::1") + end + + assert_includes exception.message, "Invalid publish IP address: 1.2.3.invalidIP" + end + test "boot_config set docker options" do run_command("boot_config", "set", "--docker_options", "label=foo=bar", "add_host=thishost:thathost").tap do |output| %w[ 1.1.1.1 1.1.1.2 ].each do |host| diff --git a/test/cli/registry_test.rb b/test/cli/registry_test.rb index c5423fe7..e89a15e4 100644 --- a/test/cli/registry_test.rb +++ b/test/cli/registry_test.rb @@ -43,6 +43,16 @@ class CliRegistryTest < CliTestCase end end + test "login with no docker" do + stub_setup + SSHKit::Backend::Abstract.any_instance.stubs(:execute) + .with(:docker, "--version", "&&", :docker, :buildx, "version") + .raises(SSHKit::Command::Failed.new("command not found")) + + assert_raises(Kamal::Cli::DependencyError) { run_command("login") } + end + + private def run_command(*command) stdouted { Kamal::Cli::Registry.start([ *command, "-c", "test/fixtures/deploy_with_accessories.yml" ]) } diff --git a/test/cli/secrets_test.rb b/test/cli/secrets_test.rb index bd412862..74f309f7 100644 --- a/test/cli/secrets_test.rb +++ b/test/cli/secrets_test.rb @@ -13,12 +13,6 @@ class CliSecretsTest < CliTestCase run_command("fetch", "foo", "bar", "baz", "--adapter", "test") end - test "fetch without required --account" do - assert_equal \ - "\\{\\\"foo\\\":\\\"oof\\\",\\\"bar\\\":\\\"rab\\\",\\\"baz\\\":\\\"zab\\\"\\}", - run_command("fetch", "foo", "bar", "baz", "--adapter", "test_optional_account") - end - test "extract" do assert_equal "oof", run_command("extract", "foo", "{\"foo\":\"oof\", \"bar\":\"rab\", \"baz\":\"zab\"}") end diff --git a/test/commands/accessory_test.rb b/test/commands/accessory_test.rb index b9bcca7e..6ff9902e 100644 --- a/test/commands/accessory_test.rb +++ b/test/commands/accessory_test.rb @@ -5,7 +5,9 @@ class CommandsAccessoryTest < ActiveSupport::TestCase setup_test_secrets("secrets" => "MYSQL_ROOT_PASSWORD=secret123") @config = { - service: "app", image: "dhh/app", registry: { "server" => "private.registry", "username" => "dhh", "password" => "secret" }, + service: "app", + image: "dhh/app", + registry: { "server" => "private.registry", "username" => "dhh", "password" => "secret" }, servers: [ "1.1.1.1" ], builder: { "arch" => "amd64" }, accessories: { @@ -39,6 +41,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase "busybox" => { "service" => "custom-busybox", "image" => "busybox:latest", + "registry" => { "server" => "other.registry", "username" => "user", "password" => "pw" }, "host" => "1.1.1.7", "proxy" => { "host" => "busybox.example.com" @@ -62,7 +65,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase new_command(:redis).run.join(" ") assert_equal \ - "docker run --name custom-busybox --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --env-file .kamal/apps/app/env/accessories/busybox.env --label service=\"custom-busybox\" busybox:latest", + "docker run --name custom-busybox --detach --restart unless-stopped --network kamal --log-opt max-size=\"10m\" --env-file .kamal/apps/app/env/accessories/busybox.env --label service=\"custom-busybox\" other.registry/busybox:latest", new_command(:busybox).run.join(" ") end @@ -70,7 +73,7 @@ class CommandsAccessoryTest < ActiveSupport::TestCase @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } assert_equal \ - "docker run --name custom-busybox --detach --restart unless-stopped --network kamal --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --env-file .kamal/apps/app/env/accessories/busybox.env --label service=\"custom-busybox\" busybox:latest", + "docker run --name custom-busybox --detach --restart unless-stopped --network kamal --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" --env-file .kamal/apps/app/env/accessories/busybox.env --label service=\"custom-busybox\" other.registry/busybox:latest", new_command(:busybox).run.join(" ") end @@ -100,7 +103,6 @@ class CommandsAccessoryTest < ActiveSupport::TestCase new_command(:mysql).info.join(" ") end - test "execute in new container" do assert_equal \ "docker run --rm --network kamal --env MYSQL_ROOT_HOST=\"%\" --env-file .kamal/apps/app/env/accessories/mysql.env private.registry/mysql:8.0 mysql -u root", @@ -127,8 +129,6 @@ class CommandsAccessoryTest < ActiveSupport::TestCase end end - - test "logs" do assert_equal \ "docker logs app-mysql --timestamps 2>&1", diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index 1fb59e8a..75241597 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -157,6 +157,12 @@ class CommandsAppTest < ActiveSupport::TestCase new_command.logs.join(" ") end + test "logs with container_id" do + assert_equal \ + "echo C137 | xargs docker logs --timestamps 2>&1", + new_command.logs(container_id: "C137").join(" ") + end + test "logs with since" do assert_equal \ "sh -c 'docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | xargs docker logs --timestamps --since 5m 2>&1", @@ -208,6 +214,10 @@ class CommandsAppTest < ActiveSupport::TestCase "ssh -t root@app-1 -p 22 'sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --follow 2>&1 | grep \"Completed\"'", new_command.follow_logs(host: "app-1", grep: "Completed") + assert_equal \ + "ssh -t root@app-1 -p 22 'echo ID321 | xargs docker logs --timestamps --follow 2>&1'", + new_command.follow_logs(host: "app-1", container_id: "ID321") + assert_equal \ "ssh -t root@app-1 -p 22 'sh -c '\\''docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''\\'\\'''\\''{{.ID}}'\\''\\'\\'''\\'') ; docker ps --latest --quiet --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting'\\'' | head -1 | xargs docker logs --timestamps --tail 123 --follow 2>&1'", new_command.follow_logs(host: "app-1", lines: 123) @@ -224,29 +234,43 @@ class CommandsAppTest < ActiveSupport::TestCase test "execute in new container" do assert_equal \ - "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env dhh/app:999 bin/rails db:setup", + "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:999 bin/rails db:setup", + new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ") + end + + test "execute in new container with logging" do + @config[:logging] = { "driver" => "local", "options" => { "max-size" => "100m", "max-file" => "3" } } + + assert_equal \ + "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-driver \"local\" --log-opt max-size=\"100m\" --log-opt max-file=\"3\" dhh/app:999 bin/rails db:setup", new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ") end test "execute in new container with env" do assert_equal \ - "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --env foo=\"bar\" dhh/app:999 bin/rails db:setup", + "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --env foo=\"bar\" --log-opt max-size=\"10m\" dhh/app:999 bin/rails db:setup", new_command.execute_in_new_container("bin/rails", "db:setup", env: { "foo" => "bar" }).join(" ") end + test "execute in new detached container" do + assert_equal \ + "docker run --detach --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:999 bin/rails db:setup", + new_command.execute_in_new_container("bin/rails", "db:setup", detach: true, env: {}).join(" ") + end + test "execute in new container with tags" do @config[:servers] = [ { "1.1.1.1" => "tag1" } ] @config[:env]["tags"] = { "tag1" => { "ENV1" => "value1" } } assert_equal \ - "docker run --rm --network kamal --env ENV1=\"value1\" --env-file .kamal/apps/app/env/roles/web.env dhh/app:999 bin/rails db:setup", + "docker run --rm --network kamal --env ENV1=\"value1\" --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:999 bin/rails db:setup", new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ") end test "execute in new container with custom options" do @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } } assert_equal \ - "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails db:setup", + "docker run --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" --mount \"somewhere\" --cap-add dhh/app:999 bin/rails db:setup", new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ") end @@ -263,7 +287,7 @@ class CommandsAppTest < ActiveSupport::TestCase end test "execute in new container over ssh" do - assert_match %r{docker run -it --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env dhh/app:999 bin/rails c}, + assert_match %r{docker run -it --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size="10m" dhh/app:999 bin/rails c}, new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {}) end @@ -271,13 +295,13 @@ class CommandsAppTest < ActiveSupport::TestCase @config[:servers] = [ { "1.1.1.1" => "tag1" } ] @config[:env]["tags"] = { "tag1" => { "ENV1" => "value1" } } - assert_equal "ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --network kamal --env ENV1=\"value1\" --env-file .kamal/apps/app/env/roles/web.env dhh/app:999 bin/rails c'", + assert_equal "ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --network kamal --env ENV1=\"value1\" --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" dhh/app:999 bin/rails c'", new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {}) end test "execute in new container with custom options over ssh" do @config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } } - assert_match %r{docker run -it --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails c}, + assert_match %r{docker run -it --rm --network kamal --env-file .kamal/apps/app/env/roles/web.env --log-opt max-size=\"10m\" --mount \"somewhere\" --cap-add dhh/app:999 bin/rails c}, new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {}) end @@ -315,6 +339,16 @@ class CommandsAppTest < ActiveSupport::TestCase assert_equal "ssh -J root@2.2.2.2 -t app@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1") end + test "run over ssh with keys config" do + @config[:ssh] = { "keys" => [ "path_to_key.pem" ] } + assert_equal "ssh -i path_to_key.pem -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1") + end + + test "run over ssh with keys config with keys_only" do + @config[:ssh] = { "keys" => [ "path_to_key.pem" ], "keys_only" => true } + assert_equal "ssh -i path_to_key.pem -o IdentitiesOnly=yes -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1") + end + test "run over ssh with proxy_command" do @config[:ssh] = { "proxy_command" => "ssh -W %h:%p user@proxy-server" } assert_equal "ssh -o ProxyCommand='ssh -W %h:%p user@proxy-server' -t root@1.1.1.1 -p 22 'ls'", new_command.run_over_ssh("ls", host: "1.1.1.1") @@ -435,10 +469,10 @@ class CommandsAppTest < ActiveSupport::TestCase test "extract assets" do assert_equal [ :mkdir, "-p", ".kamal/apps/app/assets/extracted/web-999", "&&", - :docker, :stop, "-t 1", "app-web-assets", "2> /dev/null", "|| true", "&&", - :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "--entrypoint", "sleep", "dhh/app:999", "1000000", "&&", - :docker, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/apps/app/assets/extracted/web-999", "&&", - :docker, :stop, "-t 1", "app-web-assets" + :docker, :container, :rm, "app-web-assets", "2> /dev/null", "|| true", "&&", + :docker, :container, :create, "--name", "app-web-assets", "dhh/app:999", "&&", + :docker, :container, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/apps/app/assets/extracted/web-999", "&&", + :docker, :container, :rm, "app-web-assets" ], new_command(asset_path: "/public/assets").extract_assets end diff --git a/test/commands/builder_test.rb b/test/commands/builder_test.rb index 598f8a18..e1a9261c 100644 --- a/test/commands/builder_test.rb +++ b/test/commands/builder_test.rb @@ -87,6 +87,14 @@ class CommandsBuilderTest < ActiveSupport::TestCase end end + test "cloud builder" do + builder = new_builder_command(builder: { "arch" => [ "#{local_arch}" ], "driver" => "cloud docker-org-name/builder-name" }) + assert_equal "cloud", builder.name + assert_equal \ + "docker buildx build --push --platform linux/#{local_arch} --builder cloud-docker-org-name-builder-name -t dhh/app:123 -t dhh/app:latest --label service=\"app\" --file Dockerfile .", + builder.push.join(" ") + end + test "build args" do builder = new_builder_command(builder: { "args" => { "a" => 1, "b" => 2 } }) assert_equal \ diff --git a/test/commands/registry_test.rb b/test/commands/registry_test.rb index cf2734b7..0a71b1da 100755 --- a/test/commands/registry_test.rb +++ b/test/commands/registry_test.rb @@ -2,14 +2,27 @@ require "test_helper" class CommandsRegistryTest < ActiveSupport::TestCase setup do - @config = { service: "app", + @config = { + service: "app", image: "dhh/app", - registry: { "username" => "dhh", + registry: { + "username" => "dhh", "password" => "secret", "server" => "hub.docker.com" }, builder: { "arch" => "amd64" }, - servers: [ "1.1.1.1" ] + servers: [ "1.1.1.1" ], + accessories: { + "db" => { + "image" => "mysql:8.0", + "hosts" => [ "1.1.1.1" ], + "registry" => { + "username" => "user", + "password" => "pw", + "server" => "other.hub.docker.com" + } + } + } } end @@ -19,13 +32,24 @@ class CommandsRegistryTest < ActiveSupport::TestCase registry.login.join(" ") end + test "given registry login" do + assert_equal \ + "docker login other.hub.docker.com -u \"user\" -p \"pw\"", + registry.login(registry_config: accessory_registry_config).join(" ") + end + test "registry login with ENV password" do - with_test_secrets("secrets" => "KAMAL_REGISTRY_PASSWORD=more-secret") do + with_test_secrets("secrets" => "KAMAL_REGISTRY_PASSWORD=more-secret\nKAMAL_MYSQL_REGISTRY_PASSWORD=secret-pw") do @config[:registry]["password"] = [ "KAMAL_REGISTRY_PASSWORD" ] + @config[:accessories]["db"]["registry"]["password"] = [ "KAMAL_MYSQL_REGISTRY_PASSWORD" ] assert_equal \ "docker login hub.docker.com -u \"dhh\" -p \"more-secret\"", registry.login.join(" ") + + assert_equal \ + "docker login other.hub.docker.com -u \"user\" -p \"secret-pw\"", + registry.login(registry_config: accessory_registry_config).join(" ") end end @@ -55,8 +79,22 @@ class CommandsRegistryTest < ActiveSupport::TestCase registry.logout.join(" ") end + test "given registry logout" do + assert_equal \ + "docker logout other.hub.docker.com", + registry.logout(registry_config: accessory_registry_config).join(" ") + end + private def registry - Kamal::Commands::Registry.new Kamal::Configuration.new(@config) + Kamal::Commands::Registry.new main_config + end + + def main_config + Kamal::Configuration.new(@config) + end + + def accessory_registry_config + main_config.accessory("db").registry end end diff --git a/test/configuration/accessory_test.rb b/test/configuration/accessory_test.rb index d15a48ad..fc01dc90 100644 --- a/test/configuration/accessory_test.rb +++ b/test/configuration/accessory_test.rb @@ -3,7 +3,9 @@ require "test_helper" class ConfigurationAccessoryTest < ActiveSupport::TestCase setup do @deploy = { - service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" }, + service: "app", + image: "dhh/app", + registry: { "username" => "dhh", "password" => "secret" }, servers: { "web" => [ "1.1.1.1", "1.1.1.2" ], "workers" => [ "1.1.1.3", "1.1.1.4" ] @@ -12,7 +14,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase env: { "REDIS_URL" => "redis://x/y" }, accessories: { "mysql" => { - "image" => "mysql:8.0", + "image" => "public.registry/mysql:8.0", "host" => "1.1.1.5", "port" => "3306", "env" => { @@ -52,6 +54,7 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase "monitoring" => { "service" => "custom-monitoring", "image" => "monitoring:latest", + "registry" => { "server" => "other.registry", "username" => "user", "password" => "pw" }, "roles" => [ "web" ], "port" => "4321:4321", "labels" => { @@ -80,6 +83,21 @@ class ConfigurationAccessoryTest < ActiveSupport::TestCase assert_equal "custom-monitoring", @config.accessory(:monitoring).service_name end + test "image" do + assert_equal "public.registry/mysql:8.0", @config.accessory(:mysql).image + assert_equal "redis:latest", @config.accessory(:redis).image + assert_equal "other.registry/monitoring:latest", @config.accessory(:monitoring).image + end + + test "registry" do + assert_nil @config.accessory(:mysql).registry + assert_nil @config.accessory(:redis).registry + monitoring_registry = @config.accessory(:monitoring).registry + assert_equal "other.registry", monitoring_registry.server + assert_equal "user", monitoring_registry.username + assert_equal "pw", monitoring_registry.password + end + test "port" do assert_equal "3306:3306", @config.accessory(:mysql).port assert_equal "6379:6379", @config.accessory(:redis).port diff --git a/test/fixtures/deploy.elsewhere.yml b/test/fixtures/deploy.elsewhere.yml new file mode 100644 index 00000000..479e0f9d --- /dev/null +++ b/test/fixtures/deploy.elsewhere.yml @@ -0,0 +1,12 @@ +service: app3 +image: dhh/app3 +servers: + - "1.1.1.3" + - "1.1.1.4" +registry: + username: user + password: pw +builder: + arch: amd64 +aliases: + other_config: config -c config/deploy2.yml diff --git a/test/fixtures/deploy.yml b/test/fixtures/deploy.yml new file mode 100644 index 00000000..c532000f --- /dev/null +++ b/test/fixtures/deploy.yml @@ -0,0 +1,13 @@ +service: app +image: dhh/app +servers: + - "1.1.1.1" + - "1.1.1.2" +registry: + username: user + password: pw +builder: + arch: amd64 +aliases: + other_config: config -c config/deploy2.yml + other_destination_config: config -d elsewhere diff --git a/test/fixtures/deploy2.yml b/test/fixtures/deploy2.yml new file mode 100644 index 00000000..14b56951 --- /dev/null +++ b/test/fixtures/deploy2.yml @@ -0,0 +1,12 @@ +service: app2 +image: dhh/app2 +servers: + - "1.1.1.1" + - "1.1.1.2" +registry: + username: user2 + password: pw2 +builder: + arch: amd64 +aliases: + other_config: config -c config/deploy2.yml diff --git a/test/fixtures/deploy_with_accessories_with_different_registries.yml b/test/fixtures/deploy_with_accessories_with_different_registries.yml new file mode 100644 index 00000000..8bd8f1d9 --- /dev/null +++ b/test/fixtures/deploy_with_accessories_with_different_registries.yml @@ -0,0 +1,47 @@ +service: app +image: dhh/app +servers: + web: + - "1.1.1.1" + - "1.1.1.2" + workers: + - "1.1.1.3" + - "1.1.1.4" +registry: + server: private.registry + username: user + password: pw +builder: + arch: amd64 + +accessories: + mysql: + image: private.registry/mysql:5.7 + host: 1.1.1.3 + port: 3306 + env: + clear: + MYSQL_ROOT_HOST: '%' + secret: + - MYSQL_ROOT_PASSWORD + files: + - test/fixtures/files/my.cnf:/etc/mysql/my.cnf + directories: + - data:/var/lib/mysql + redis: + image: redis:latest + roles: + - web + port: 6379 + directories: + - data:/data + busybox: + service: custom-box + image: busybox:latest + host: 1.1.1.3 + registry: + server: other.registry + username: other_user + password: other_pw + +readiness_delay: 0 diff --git a/test/fixtures/deploy_with_aliases.yml b/test/fixtures/deploy_with_aliases.yml index ec7b14a0..104fc462 100644 --- a/test/fixtures/deploy_with_aliases.yml +++ b/test/fixtures/deploy_with_aliases.yml @@ -21,3 +21,6 @@ aliases: console: app exec --reuse -p -r console "bin/console" exec: app exec --reuse -p -r console rails: app exec --reuse -p -r console rails + primary_details: details -p + deploy_secondary: deploy -d secondary + diff --git a/test/fixtures/deploy_with_cloud_builder.yml b/test/fixtures/deploy_with_cloud_builder.yml new file mode 100644 index 00000000..420ff205 --- /dev/null +++ b/test/fixtures/deploy_with_cloud_builder.yml @@ -0,0 +1,40 @@ +service: app +image: dhh/app +servers: + web: + - "1.1.1.1" + - "1.1.1.2" + workers: + - "1.1.1.3" + - "1.1.1.4" +registry: + username: user + password: pw + +accessories: + mysql: + image: mysql:5.7 + host: 1.1.1.3 + port: 3306 + env: + clear: + MYSQL_ROOT_HOST: '%' + secret: + - MYSQL_ROOT_PASSWORD + files: + - test/fixtures/files/my.cnf:/etc/mysql/my.cnf + directories: + - data:/var/lib/mysql + redis: + image: redis:latest + roles: + - web + port: 6379 + directories: + - data:/data + +readiness_delay: 0 + +builder: + arch: <%= Kamal::Utils.docker_arch == "arm64" ? "amd64" : "arm64" %> + driver: cloud example_org/cloud_builder diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb index ce32e640..a48051fe 100644 --- a/test/integration/main_test.rb +++ b/test/integration/main_test.rb @@ -90,9 +90,9 @@ class MainTest < IntegrationTest test "setup and remove" do @app = "app_with_roles" - kamal :proxy, :set_config, + kamal :proxy, :boot_config, "set", "--publish=false", - "--options=label=traefik.http.services.kamal_proxy.loadbalancer.server.scheme=http", + "--docker-options=label=traefik.http.services.kamal_proxy.loadbalancer.server.scheme=http", "label=traefik.http.routers.kamal_proxy.rule=PathPrefix\\\(\\\`/\\\`\\\)", "label=traefik.http.routers.kamal_proxy.priority=2" diff --git a/test/secrets/aws_secrets_manager_adapter_test.rb b/test/secrets/aws_secrets_manager_adapter_test.rb index 42a0f48a..7616342d 100644 --- a/test/secrets/aws_secrets_manager_adapter_test.rb +++ b/test/secrets/aws_secrets_manager_adapter_test.rb @@ -1,6 +1,35 @@ require "test_helper" class AwsSecretsManagerAdapterTest < SecretAdapterTestCase + test "fails when errors are present" do + stub_ticks.with("aws --version 2> /dev/null") + stub_ticks + .with("aws secretsmanager batch-get-secret-value --secret-id-list unknown1 unknown2 --profile default") + .returns(<<~JSON) + { + "SecretValues": [], + "Errors": [ + { + "SecretId": "unknown1", + "ErrorCode": "ResourceNotFoundException", + "Message": "Secrets Manager can't find the specified secret." + }, + { + "SecretId": "unknown2", + "ErrorCode": "ResourceNotFoundException", + "Message": "Secrets Manager can't find the specified secret." + } + ] + } + JSON + + error = assert_raises RuntimeError do + JSON.parse(shellunescape(run_command("fetch", "unknown1", "unknown2"))) + end + + assert_equal [ "unknown1: Secrets Manager can't find the specified secret.", "unknown2: Secrets Manager can't find the specified secret." ].join(" "), error.message + end + test "fetch" do stub_ticks.with("aws --version 2> /dev/null") stub_ticks @@ -44,6 +73,48 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase assert_equal expected_json, json end + test "fetch with string value" do + stub_ticks.with("aws --version 2> /dev/null") + stub_ticks + .with("aws secretsmanager batch-get-secret-value --secret-id-list secret secret2/KEY1 --profile default") + .returns(<<~JSON) + { + "SecretValues": [ + { + "ARN": "arn:aws:secretsmanager:us-east-1:aaaaaaaaaaaa:secret:secret", + "Name": "secret", + "VersionId": "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv", + "SecretString": "a-string-secret", + "VersionStages": [ + "AWSCURRENT" + ], + "CreatedDate": "2024-01-01T00:00:00.000000" + }, + { + "ARN": "arn:aws:secretsmanager:us-east-1:aaaaaaaaaaaa:secret:secret2", + "Name": "secret2", + "VersionId": "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv", + "SecretString": "{\\"KEY2\\":\\"VALUE2\\"}", + "VersionStages": [ + "AWSCURRENT" + ], + "CreatedDate": "2024-01-01T00:00:00.000000" + } + ], + "Errors": [] + } + JSON + + json = JSON.parse(shellunescape(run_command("fetch", "secret", "secret2/KEY1"))) + + expected_json = { + "secret"=>"a-string-secret", + "secret2/KEY2"=>"VALUE2" + } + + assert_equal expected_json, json + end + test "fetch with secret names" do stub_ticks.with("aws --version 2> /dev/null") stub_ticks diff --git a/test/secrets/bitwarden_secrets_manager_adapter_test.rb b/test/secrets/bitwarden_secrets_manager_adapter_test.rb new file mode 100644 index 00000000..1723da42 --- /dev/null +++ b/test/secrets/bitwarden_secrets_manager_adapter_test.rb @@ -0,0 +1,119 @@ +require "test_helper" + +class BitwardenSecretsManagerAdapterTest < SecretAdapterTestCase + test "fetch with no parameters" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + + error = assert_raises RuntimeError do + (shellunescape(run_command("fetch"))) + end + assert_equal("You must specify what to retrieve from Bitwarden Secrets Manager", error.message) + end + + test "fetch all" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks + .with("bws secret list -o env") + .returns("KAMAL_REGISTRY_PASSWORD=\"some_password\"\nMY_OTHER_SECRET=\"my=weird\"secret\"") + + expected = '{"KAMAL_REGISTRY_PASSWORD":"some_password","MY_OTHER_SECRET":"my\=weird\"secret"}' + actual = shellunescape(run_command("fetch", "all")) + assert_equal expected, actual + end + + test "fetch all with from" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks + .with("bws secret list -o env 82aeb5bd-6958-4a89-8197-eacab758acce") + .returns("KAMAL_REGISTRY_PASSWORD=\"some_password\"\nMY_OTHER_SECRET=\"my=weird\"secret\"") + + expected = '{"KAMAL_REGISTRY_PASSWORD":"some_password","MY_OTHER_SECRET":"my\=weird\"secret"}' + actual = shellunescape(run_command("fetch", "all", "--from", "82aeb5bd-6958-4a89-8197-eacab758acce")) + assert_equal expected, actual + end + + test "fetch item" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks + .with("bws secret get -o env 82aeb5bd-6958-4a89-8197-eacab758acce") + .returns("KAMAL_REGISTRY_PASSWORD=\"some_password\"") + + expected = '{"KAMAL_REGISTRY_PASSWORD":"some_password"}' + actual = shellunescape(run_command("fetch", "82aeb5bd-6958-4a89-8197-eacab758acce")) + assert_equal expected, actual + end + + test "fetch with multiple items" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks + .with("bws secret get -o env 82aeb5bd-6958-4a89-8197-eacab758acce") + .returns("KAMAL_REGISTRY_PASSWORD=\"some_password\"") + stub_ticks + .with("bws secret get -o env 6f8cdf27-de2b-4c77-a35d-07df8050e332") + .returns("MY_OTHER_SECRET=\"my=weird\"secret\"") + + expected = '{"KAMAL_REGISTRY_PASSWORD":"some_password","MY_OTHER_SECRET":"my\=weird\"secret"}' + actual = shellunescape(run_command("fetch", "82aeb5bd-6958-4a89-8197-eacab758acce", "6f8cdf27-de2b-4c77-a35d-07df8050e332")) + assert_equal expected, actual + end + + test "fetch all empty" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks_with("bws secret list -o env", succeed: false).returns("Error:\n0: Received error message from server") + + error = assert_raises RuntimeError do + (shellunescape(run_command("fetch", "all"))) + end + assert_equal("Could not read secrets from Bitwarden Secrets Manager", error.message) + end + + test "fetch nonexistent item" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks_with("bws secret get -o env 82aeb5bd-6958-4a89-8197-eacab758acce", succeed: false) + .returns("ERROR (RuntimeError): Could not read 82aeb5bd-6958-4a89-8197-eacab758acce from Bitwarden Secrets Manager") + + error = assert_raises RuntimeError do + (shellunescape(run_command("fetch", "82aeb5bd-6958-4a89-8197-eacab758acce"))) + end + assert_equal("Could not read 82aeb5bd-6958-4a89-8197-eacab758acce from Bitwarden Secrets Manager", error.message) + end + + test "fetch with no access token" do + stub_ticks.with("bws --version 2> /dev/null") + stub_ticks_with("bws run 'echo OK'", succeed: false) + + error = assert_raises RuntimeError do + (shellunescape(run_command("fetch", "all"))) + end + assert_equal("Could not authenticate to Bitwarden Secrets Manager. Did you set a valid access token?", error.message) + end + + test "fetch without CLI installed" do + stub_ticks_with("bws --version 2> /dev/null", succeed: false) + + error = assert_raises RuntimeError do + shellunescape(run_command("fetch")) + end + assert_equal "Bitwarden Secrets Manager CLI is not installed", error.message + end + + private + def stub_login + stub_ticks.with("bws run 'echo OK'").returns("OK") + end + + def run_command(*command) + stdouted do + Kamal::Cli::Secrets.start \ + [ *command, + "--adapter", "bitwarden-sm" ] + end + end +end diff --git a/test/secrets/enpass_adapter_test.rb b/test/secrets/enpass_adapter_test.rb new file mode 100644 index 00000000..edc49613 --- /dev/null +++ b/test/secrets/enpass_adapter_test.rb @@ -0,0 +1,81 @@ +require "test_helper" + +class EnpassAdapterTest < SecretAdapterTestCase + test "fetch without CLI installed" do + stub_ticks_with("enpass-cli version 2> /dev/null", succeed: false) + + error = assert_raises RuntimeError do + JSON.parse(shellunescape(run_command("fetch", "mynote"))) + end + + assert_equal "Enpass CLI is not installed", error.message + end + + test "fetch one item" do + stub_ticks_with("enpass-cli version 2> /dev/null") + + stub_ticks + .with("enpass-cli -json -vault vault-path show FooBar") + .returns(<<~JSON) + [{"category":"computer","label":"SECRET_1","login":"","password":"my-password-1","title":"FooBar","type":"password"}] + JSON + + json = JSON.parse(shellunescape(run_command("fetch", "FooBar/SECRET_1"))) + + expected_json = { "FooBar/SECRET_1" => "my-password-1" } + + assert_equal expected_json, json + end + + test "fetch multiple items" do + stub_ticks_with("enpass-cli version 2> /dev/null") + + stub_ticks + .with("enpass-cli -json -vault vault-path show FooBar") + .returns(<<~JSON) + [ + {"category":"computer","label":"SECRET_1","login":"","password":"my-password-1","title":"FooBar","type":"password"}, + {"category":"computer","label":"SECRET_2","login":"","password":"my-password-2","title":"FooBar","type":"password"}, + {"category":"computer","label":"SECRET_3","login":"","password":"my-password-1","title":"Hello","type":"password"} + ] + JSON + + json = JSON.parse(shellunescape(run_command("fetch", "FooBar/SECRET_1", "FooBar/SECRET_2"))) + + expected_json = { "FooBar/SECRET_1" => "my-password-1", "FooBar/SECRET_2" => "my-password-2" } + + assert_equal expected_json, json + end + + test "fetch all with from" do + stub_ticks_with("enpass-cli version 2> /dev/null") + + stub_ticks + .with("enpass-cli -json -vault vault-path show FooBar") + .returns(<<~JSON) + [ + {"category":"computer","label":"SECRET_1","login":"","password":"my-password-1","title":"FooBar","type":"password"}, + {"category":"computer","label":"SECRET_2","login":"","password":"my-password-2","title":"FooBar","type":"password"}, + {"category":"computer","label":"SECRET_3","login":"","password":"my-password-1","title":"Hello","type":"password"}, + {"category":"computer","label":"","login":"","password":"my-password-3","title":"FooBar","type":"password"} + ] + JSON + + json = JSON.parse(shellunescape(run_command("fetch", "FooBar"))) + + expected_json = { "FooBar/SECRET_1" => "my-password-1", "FooBar/SECRET_2" => "my-password-2", "FooBar" => "my-password-3" } + + assert_equal expected_json, json + end + + private + def run_command(*command) + stdouted do + Kamal::Cli::Secrets.start \ + [ *command, + "-c", "test/fixtures/deploy_with_accessories.yml", + "--adapter", "enpass", + "--from", "vault-path" ] + end + end +end diff --git a/test/secrets/gcp_secret_manager_adapter_test.rb b/test/secrets/gcp_secret_manager_adapter_test.rb new file mode 100644 index 00000000..682db1f4 --- /dev/null +++ b/test/secrets/gcp_secret_manager_adapter_test.rb @@ -0,0 +1,220 @@ +require "test_helper" + +class GcpSecretManagerAdapterTest < SecretAdapterTestCase + test "fetch" do + stub_gcloud_version + stub_authenticated + stub_mypassword + + json = JSON.parse(shellunescape(run_command("fetch", "mypassword"))) + + expected_json = { "default/mypassword"=>"secret123" } + + assert_equal expected_json, json + end + + test "fetch unauthenticated" do + stub_ticks.with("gcloud --version 2> /dev/null") + + stub_mypassword + stub_unauthenticated + + error = assert_raises RuntimeError do + JSON.parse(shellunescape(run_command("fetch", "mypassword"))) + end + + assert_match(/could not login to gcloud/, error.message) + end + + test "fetch with from" do + stub_gcloud_version + stub_authenticated + stub_items(0, project: "other-project") + stub_items(1, project: "other-project") + stub_items(2, project: "other-project") + + json = JSON.parse(shellunescape(run_command("fetch", "--from", "other-project", "item1", "item2", "item3"))) + + expected_json = { + "other-project/item1"=>"secret1", "other-project/item2"=>"secret2", "other-project/item3"=>"secret3" + } + + assert_equal expected_json, json + end + + test "fetch with multiple projects" do + stub_gcloud_version + stub_authenticated + stub_items(0, project: "some-project") + stub_items(1, project: "project-confidence") + stub_items(2, project: "manhattan-project") + + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1", "project-confidence/item2", "manhattan-project/item3"))) + + expected_json = { + "some-project/item1"=>"secret1", "project-confidence/item2"=>"secret2", "manhattan-project/item3"=>"secret3" + } + + assert_equal expected_json, json + end + + test "fetch with specific version" do + stub_gcloud_version + stub_authenticated + stub_items(0, project: "some-project", version: "123") + + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123"))) + + expected_json = { + "some-project/item1"=>"secret1" + } + + assert_equal expected_json, json + end + + test "fetch with non-default account" do + stub_gcloud_version + stub_authenticated + stub_items(0, project: "some-project", version: "123", account: "email@example.com") + + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123", account: "email@example.com"))) + + expected_json = { + "some-project/item1"=>"secret1" + } + + assert_equal expected_json, json + end + + test "fetch with service account impersonation" do + stub_gcloud_version + stub_authenticated + stub_items(0, project: "some-project", version: "123", impersonate_service_account: "service-user@example.com") + + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123", account: "default|service-user@example.com"))) + + expected_json = { + "some-project/item1"=>"secret1" + } + + assert_equal expected_json, json + end + + test "fetch with delegation chain and specific user" do + stub_gcloud_version + stub_authenticated + stub_items(0, project: "some-project", version: "123", account: "user@example.com", impersonate_service_account: "service-user@example.com,service-user2@example.com") + + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123", account: "user@example.com|service-user@example.com,service-user2@example.com"))) + + expected_json = { + "some-project/item1"=>"secret1" + } + + assert_equal expected_json, json + end + + test "fetch with non-default account and service account impersonation" do + stub_gcloud_version + stub_authenticated + stub_items(0, project: "some-project", version: "123", account: "email@example.com", impersonate_service_account: "service-user@example.com") + + json = JSON.parse(shellunescape(run_command("fetch", "some-project/item1/123", account: "email@example.com|service-user@example.com"))) + + expected_json = { + "some-project/item1"=>"secret1" + } + + assert_equal expected_json, json + end + + test "fetch without CLI installed" do + stub_gcloud_version(succeed: false) + + error = assert_raises RuntimeError do + JSON.parse(shellunescape(run_command("fetch", "item1"))) + end + assert_equal "gcloud CLI is not installed", error.message + end + + private + def run_command(*command, account: "default") + stdouted do + Kamal::Cli::Secrets.start \ + [ *command, + "-c", "test/fixtures/deploy_with_accessories.yml", + "--adapter", "gcp_secret_manager", + "--account", account ] + end + end + + def stub_gcloud_version(succeed: true) + stub_ticks_with("gcloud --version 2> /dev/null", succeed: succeed) + end + + def stub_authenticated + stub_ticks + .with("gcloud auth list --format=json") + .returns(<<~JSON) + [ + { + "account": "email@example.com", + "status": "ACTIVE" + } + ] + JSON + end + + def stub_unauthenticated + stub_ticks + .with("gcloud auth list --format=json") + .returns("[]") + + stub_ticks + .with("gcloud auth login") + .returns(<<~JSON) + { + "expired": false, + "valid": true + } + JSON + end + + def stub_mypassword + stub_ticks + .with("gcloud secrets versions access latest --secret=mypassword --format=json") + .returns(<<~JSON) + { + "name": "projects/000000000/secrets/mypassword/versions/1", + "payload": { + "data": "c2VjcmV0MTIz", + "dataCrc32c": "2522602764" + } + } + JSON + end + + def stub_items(n, project: nil, account: nil, version: "latest", impersonate_service_account: nil) + payloads = [ + { data: "c2VjcmV0MQ==", checksum: 1846998209 }, + { data: "c2VjcmV0Mg==", checksum: 2101741365 }, + { data: "c2VjcmV0Mw==", checksum: 2402124854 } + ] + stub_ticks + .with("gcloud secrets versions access #{version} " \ + "--secret=item#{n + 1}" \ + "#{" --project=#{project}" if project}" \ + "#{" --account=#{account}" if account}" \ + "#{" --impersonate-service-account=#{impersonate_service_account}" if impersonate_service_account} " \ + "--format=json") + .returns(<<~JSON) + { + "name": "projects/000000001/secrets/item1/versions/1", + "payload": { + "data": "#{payloads[n][:data]}", + "dataCrc32c": "#{payloads[n][:checksum]}" + } + } + JSON + end +end diff --git a/test/secrets_test.rb b/test/secrets_test.rb index aca9cebe..f0ca7e29 100644 --- a/test/secrets_test.rb +++ b/test/secrets_test.rb @@ -20,6 +20,20 @@ class SecretsTest < ActiveSupport::TestCase end end + test "env references" do + with_test_secrets("secrets" => "SECRET1=$SECRET1") do + ENV["SECRET1"] = "ABC" + assert_equal "ABC", Kamal::Secrets.new["SECRET1"] + end + end + + test "secrets file value overrides env" do + with_test_secrets("secrets" => "SECRET1=DEF") do + ENV["SECRET1"] = "ABC" + assert_equal "DEF", Kamal::Secrets.new["SECRET1"] + end + end + test "destinations" do with_test_secrets("secrets.dest" => "SECRET=DEF", "secrets" => "SECRET=ABC", "secrets-common" => "SECRET=GHI\nSECRET2=JKL") do assert_equal "ABC", Kamal::Secrets.new["SECRET"]