diff --git a/Gemfile.lock b/Gemfile.lock index 9af5e108..67963d62 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - kamal (2.3.0) + kamal (2.4.0) activesupport (>= 7.0) base64 (~> 0.2) bcrypt_pbkdf (~> 1.0) @@ -16,80 +16,87 @@ PATH GEM remote: https://rubygems.org/ specs: - actionpack (7.1.4.1) - actionview (= 7.1.4.1) - activesupport (= 7.1.4.1) + actionpack (8.0.0.1) + actionview (= 8.0.0.1) + activesupport (= 8.0.0.1) nokogiri (>= 1.8.5) - racc rack (>= 2.2.4) rack-session (>= 1.0.1) rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - actionview (7.1.4.1) - activesupport (= 7.1.4.1) + useragent (~> 0.16) + actionview (8.0.0.1) + activesupport (= 8.0.0.1) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activesupport (7.1.4.1) + activesupport (8.0.0.1) base64 + benchmark (>= 0.3) bigdecimal - concurrent-ruby (~> 1.0, >= 1.0.2) + concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) + logger (>= 1.4.2) minitest (>= 5.1) - mutex_m - tzinfo (~> 2.0) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) ast (2.4.2) base64 (0.2.0) bcrypt_pbkdf (1.1.1) bcrypt_pbkdf (1.1.1-arm64-darwin) bcrypt_pbkdf (1.1.1-x86_64-darwin) + benchmark (0.4.0) bigdecimal (3.1.8) builder (3.3.0) concurrent-ruby (1.3.4) connection_pool (2.4.1) crass (1.0.6) + date (3.4.1) debug (1.9.2) irb (~> 1.10) reline (>= 0.3.8) - dotenv (3.1.2) + dotenv (3.1.5) drb (2.2.1) ed25519 (1.3.0) erubi (1.13.0) i18n (1.14.6) concurrent-ruby (~> 1.0) - io-console (0.7.2) - irb (1.14.0) + io-console (0.8.0) + irb (1.14.2) rdoc (>= 4.0.0) reline (>= 0.4.2) - json (2.7.2) + json (2.9.0) language_server-protocol (3.17.0.3) + logger (1.6.3) loofah (2.23.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) - minitest (5.25.1) - mocha (2.4.5) + minitest (5.25.4) + mocha (2.7.1) ruby2_keywords (>= 0.0.5) - mutex_m (0.2.0) net-scp (4.0.0) net-ssh (>= 2.6.5, < 8.0.0) net-sftp (4.0.0) net-ssh (>= 5.0.0, < 8.0.0) net-ssh (7.3.0) - nokogiri (1.16.8-arm64-darwin) + nokogiri (1.17.2-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.8-x86_64-darwin) + nokogiri (1.17.2-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.8-x86_64-linux) + nokogiri (1.17.2-x86_64-linux) racc (~> 1.4) - parallel (1.25.1) - parser (3.3.4.0) + ostruct (0.6.1) + parallel (1.26.3) + parser (3.3.6.0) ast (~> 2.4.1) racc - psych (5.1.2) + psych (5.2.1) + date stringio racc (1.8.1) rack (3.1.8) @@ -97,55 +104,52 @@ GEM rack (>= 3.0.0) rack-test (2.1.0) rack (>= 1.3) - rackup (2.1.0) + rackup (2.2.1) rack (>= 3) - webrick (~> 1.8) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.6.1) + rails-html-sanitizer (1.6.2) loofah (~> 2.21) nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) - railties (7.1.4.1) - actionpack (= 7.1.4.1) - activesupport (= 7.1.4.1) - irb + railties (8.0.0.1) + actionpack (= 8.0.0.1) + activesupport (= 8.0.0.1) + irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) thor (~> 1.0, >= 1.2.2) zeitwerk (~> 2.6) rainbow (3.1.1) rake (13.2.1) - rdoc (6.7.0) + rdoc (6.8.1) psych (>= 4.0.0) - regexp_parser (2.9.2) - reline (0.5.9) + regexp_parser (2.9.3) + reline (0.5.12) io-console (~> 0.5) - rexml (3.3.9) - rubocop (1.65.1) + rubocop (1.69.2) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 2.4, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.31.1, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.36.2, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.32.0) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.36.2) parser (>= 3.3.1.0) - rubocop-minitest (0.35.1) + rubocop-minitest (0.36.0) rubocop (>= 1.61, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) - rubocop-performance (1.21.1) + rubocop-performance (1.23.0) rubocop (>= 1.48.1, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) - rubocop-rails (2.25.1) + rubocop-rails (2.27.0) activesupport (>= 4.2.0) rack (>= 1.1) - rubocop (>= 1.33.0, < 2.0) + rubocop (>= 1.52.0, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) rubocop-rails-omakase (1.0.0) rubocop @@ -154,17 +158,22 @@ GEM rubocop-rails ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) - sshkit (1.23.0) + securerandom (0.4.0) + sshkit (1.23.2) base64 net-scp (>= 1.1.2) net-sftp (>= 2.1.2) net-ssh (>= 2.8.0) - stringio (3.1.1) - thor (1.3.1) + ostruct + stringio (3.1.2) + thor (1.3.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - unicode-display_width (2.5.0) - webrick (1.8.2) + unicode-display_width (3.1.2) + unicode-emoji (~> 4.0, >= 4.0.4) + unicode-emoji (4.0.4) + uri (1.0.2) + useragent (0.16.11) zeitwerk (2.7.1) PLATFORMS diff --git a/lib/kamal/cli/accessory.rb b/lib/kamal/cli/accessory.rb index 4de8832b..00999b2d 100644 --- a/lib/kamal/cli/accessory.rb +++ b/lib/kamal/cli/accessory.rb @@ -162,7 +162,7 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base option :since, aliases: "-s", desc: "Show logs since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)" option :lines, type: :numeric, aliases: "-n", desc: "Number of log lines to pull from each server" option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)" - option :grep_options, aliases: "-o", desc: "Additional options supplied to grep" + option :grep_options, desc: "Additional options supplied to grep" option :follow, aliases: "-f", desc: "Follow logs on primary server (or specific host set by --hosts)" option :skip_timestamps, type: :boolean, aliases: "-T", desc: "Skip appending timestamps to logging output" def logs(name) diff --git a/lib/kamal/cli/app.rb b/lib/kamal/cli/app.rb index 2378fa03..fb665af9 100644 --- a/lib/kamal/cli/app.rb +++ b/lib/kamal/cli/app.rb @@ -192,7 +192,7 @@ class Kamal::Cli::App < Kamal::Cli::Base option :since, aliases: "-s", desc: "Show lines since timestamp (e.g. 2013-01-02T13:23:37Z) or relative (e.g. 42m for 42 minutes)" option :lines, type: :numeric, aliases: "-n", desc: "Number of lines to show from each server" option :grep, aliases: "-g", desc: "Show lines with grep match only (use this to fetch specific requests by id)" - option :grep_options, aliases: "-o", desc: "Additional options supplied to grep" + option :grep_options, desc: "Additional options supplied to grep" option :follow, aliases: "-f", desc: "Follow log on primary server (or specific host set by --hosts)" option :skip_timestamps, type: :boolean, aliases: "-T", desc: "Skip appending timestamps to logging output" option :container_id, desc: "Docker container ID to fetch logs" diff --git a/lib/kamal/cli/app/boot.rb b/lib/kamal/cli/app/boot.rb index 41db39a6..8ddcaac3 100644 --- a/lib/kamal/cli/app/boot.rb +++ b/lib/kamal/cli/app/boot.rb @@ -45,7 +45,7 @@ class Kamal::Cli::App::Boot def start_new_version audit "Booted app version #{version}" - hostname = "#{host.to_s[0...51].gsub(/\.+$/, '')}-#{SecureRandom.hex(6)}" + hostname = "#{host.to_s[0...51].chomp(".")}-#{SecureRandom.hex(6)}" execute *app.ensure_env_directory upload! role.secrets_io(host), role.secrets_path, mode: "0600" diff --git a/lib/kamal/cli/base.rb b/lib/kamal/cli/base.rb index 4aebfd90..74f75ffc 100644 --- a/lib/kamal/cli/base.rb +++ b/lib/kamal/cli/base.rb @@ -5,7 +5,7 @@ module Kamal::Cli class Base < Thor include SSHKit::DSL - def self.exit_on_failure?() false end + def self.exit_on_failure?() true end def self.dynamic_command_class() Kamal::Cli::Alias::Command end class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging" @@ -30,7 +30,8 @@ module Kamal::Cli else super end - initialize_commander unless KAMAL.configured? + + initialize_commander unless config[:invoked_via_subcommand] end private diff --git a/lib/kamal/cli/proxy.rb b/lib/kamal/cli/proxy.rb index d0e9ba2b..43444539 100644 --- a/lib/kamal/cli/proxy.rb +++ b/lib/kamal/cli/proxy.rb @@ -23,6 +23,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base desc "boot_config ", "Manage kamal-proxy boot configuration" option :publish, type: :boolean, default: true, desc: "Publish the proxy ports on the host" + option :publish_host_ip, type: :string, repeatable: true, default: nil, desc: "Host IP address to bind HTTP/HTTPS traffic to. Defaults to all interfaces" option :http_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTP_PORT, desc: "HTTP port to publish on the host" option :https_port, type: :numeric, default: Kamal::Configuration::PROXY_HTTPS_PORT, desc: "HTTPS port to publish on the host" option :log_max_size, type: :string, default: Kamal::Configuration::PROXY_LOG_MAX_SIZE, desc: "Max size of proxy logs" @@ -31,7 +32,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base case subcommand when "set" boot_options = [ - *(KAMAL.config.proxy_publish_args(options[:http_port], options[:https_port]) if options[:publish]), + *(KAMAL.config.proxy_publish_args(options[:http_port], options[:https_port], options[:publish_host_ip]) if options[:publish]), *(KAMAL.config.proxy_logging_args(options[:log_max_size])), *options[:docker_options].map { |option| "--#{option}" } ] diff --git a/lib/kamal/commands/app/assets.rb b/lib/kamal/commands/app/assets.rb index c1e65d18..21ae4d5f 100644 --- a/lib/kamal/commands/app/assets.rb +++ b/lib/kamal/commands/app/assets.rb @@ -4,10 +4,10 @@ module Kamal::Commands::App::Assets combine \ make_directory(role.asset_extracted_directory), - [ *docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true" ], - docker(:run, "--name", asset_container, "--detach", "--rm", "--entrypoint", "sleep", config.absolute_image, "1000000"), - docker(:cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_directory), - docker(:stop, "-t 1", asset_container), + [ *docker(:container, :rm, asset_container, "2> /dev/null"), "|| true" ], + docker(:container, :create, "--name", asset_container, config.absolute_image), + docker(:container, :cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_directory), + docker(:container, :rm, asset_container), by: "&&" end diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index 92d850e2..250eef09 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -37,7 +37,7 @@ class Kamal::Configuration if file.exist? # Newer Psych doesn't load aliases by default load_method = YAML.respond_to?(:unsafe_load) ? :unsafe_load : :load - YAML.send(load_method, ERB.new(IO.read(file)).result).symbolize_keys + YAML.send(load_method, ERB.new(File.read(file)).result).symbolize_keys else raise "Configuration file not found in #{file}" end @@ -249,8 +249,16 @@ class Kamal::Configuration env_tags.detect { |t| t.name == name.to_s } end - def proxy_publish_args(http_port, https_port) - argumentize "--publish", [ "#{http_port}:#{PROXY_HTTP_PORT}", "#{https_port}:#{PROXY_HTTPS_PORT}" ] + def proxy_publish_args(http_port, https_port, bind_ips = nil) + ensure_valid_bind_ips(bind_ips) + + (bind_ips || [ nil ]).map do |bind_ip| + bind_ip = format_bind_ip(bind_ip) + publish_http = [ bind_ip, http_port, PROXY_HTTP_PORT ].compact.join(":") + publish_https = [ bind_ip, https_port, PROXY_HTTPS_PORT ].compact.join(":") + + argumentize "--publish", [ publish_http, publish_https ] + end.join(" ") end def proxy_logging_args(max_size) @@ -344,6 +352,15 @@ class Kamal::Configuration true end + def ensure_valid_bind_ips(bind_ips) + bind_ips.present? && bind_ips.each do |ip| + next if ip =~ Resolv::IPv4::Regex || ip =~ Resolv::IPv6::Regex + raise ArgumentError, "Invalid publish IP address: #{ip}" + end + + true + end + def ensure_retain_containers_valid raise Kamal::ConfigurationError, "Must retain at least 1 container" if retain_containers < 1 @@ -375,6 +392,15 @@ class Kamal::Configuration true end + def format_bind_ip(ip) + # Ensure IPv6 address inside square brackets - e.g. [::1] + if ip =~ Resolv::IPv6::Regex && ip !~ /\[.*\]/ + "[#{ip}]" + else + ip + end + end + def role_names raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort end diff --git a/lib/kamal/configuration/accessory.rb b/lib/kamal/configuration/accessory.rb index 2728607d..198e6321 100644 --- a/lib/kamal/configuration/accessory.rb +++ b/lib/kamal/configuration/accessory.rb @@ -142,7 +142,7 @@ class Kamal::Configuration::Accessory end def read_dynamic_file(local_file) - StringIO.new(ERB.new(IO.read(local_file)).result) + StringIO.new(ERB.new(File.read(local_file)).result) end def expand_remote_file(remote_file) diff --git a/lib/kamal/configuration/docs/accessory.yml b/lib/kamal/configuration/docs/accessory.yml index fab2989f..b82a476e 100644 --- a/lib/kamal/configuration/docs/accessory.yml +++ b/lib/kamal/configuration/docs/accessory.yml @@ -43,8 +43,8 @@ accessories: # Port mappings # - # See https://docs.docker.com/network/, and especially note the warning about the security - # implications of exposing ports publicly. + # See [https://docs.docker.com/network/](https://docs.docker.com/network/), and + # especially note the warning about the security implications of exposing ports publicly. port: "127.0.0.1:3306:3306" # Labels @@ -101,4 +101,4 @@ accessories: # Proxy # proxy: - ... \ No newline at end of file + ... diff --git a/lib/kamal/configuration/docs/alias.yml b/lib/kamal/configuration/docs/alias.yml index 32c37bad..6c46b695 100644 --- a/lib/kamal/configuration/docs/alias.yml +++ b/lib/kamal/configuration/docs/alias.yml @@ -5,12 +5,12 @@ # For example, for a Rails app, you might open a console with: # # ```shell -# kamal app exec -i -r console "rails console" +# kamal app exec -i --reuse "bin/rails console" # ``` # # By defining an alias, like this: aliases: - console: app exec -r console -i "rails console" + console: app exec -i --reuse "bin/rails console" # You can now open the console with: # # ```shell diff --git a/lib/kamal/configuration/docs/proxy.yml b/lib/kamal/configuration/docs/proxy.yml index 76ec3e41..9ed6a97f 100644 --- a/lib/kamal/configuration/docs/proxy.yml +++ b/lib/kamal/configuration/docs/proxy.yml @@ -46,9 +46,22 @@ proxy: # The host value must point to the server we are deploying to, and port 443 must be # open for the Let's Encrypt challenge to succeed. # + # If you set `ssl` to `true`, `kamal-proxy` will stop forwarding headers to your app, + # unless you explicitly set `forward_headers: true` + # # Defaults to `false`: ssl: true + # Forward headers + # + # Whether to forward the `X-Forwarded-For` and `X-Forwarded-Proto` headers. + # + # If you are behind a trusted proxy, you can set this to `true` to forward the headers. + # + # By default, kamal-proxy will not forward the headers if the `ssl` option is set to `true`, and + # will forward them if it is set to `false`. + forward_headers: true + # Response timeout # # How long to wait for requests to complete before timing out, defaults to 30 seconds: @@ -93,13 +106,3 @@ proxy: response_headers: - X-Request-ID - X-Request-Start - - # Forward headers - # - # Whether to forward the `X-Forwarded-For` and `X-Forwarded-Proto` headers. - # - # If you are behind a trusted proxy, you can set this to `true` to forward the headers. - # - # By default, kamal-proxy will not forward the headers if the `ssl` option is set to `true`, and - # will forward them if it is set to `false`. - forward_headers: true diff --git a/lib/kamal/configuration/docs/registry.yml b/lib/kamal/configuration/docs/registry.yml index 84f9c7c1..4411fd4d 100644 --- a/lib/kamal/configuration/docs/registry.yml +++ b/lib/kamal/configuration/docs/registry.yml @@ -2,6 +2,10 @@ # # The default registry is Docker Hub, but you can change it using `registry/server`. # +# By default, Docker Hub creates public repositories. To avoid making your images public, +# set up a private repository before deploying, or change the default repository privacy +# settings to private in your [Docker Hub settings](https://hub.docker.com/repository-settings/default-privacy). +# # A reference to a secret (in this case, `DOCKER_REGISTRY_TOKEN`) will look up the secret # in the local environment: registry: diff --git a/lib/kamal/secrets.rb b/lib/kamal/secrets.rb index 7a382f98..9d4b0ff7 100644 --- a/lib/kamal/secrets.rb +++ b/lib/kamal/secrets.rb @@ -32,7 +32,7 @@ class Kamal::Secrets private def secrets @secrets ||= secrets_files.inject({}) do |secrets, secrets_file| - secrets.merge!(::Dotenv.parse(secrets_file)) + secrets.merge!(::Dotenv.parse(secrets_file, overwrite: true)) end end diff --git a/lib/kamal/secrets/adapters.rb b/lib/kamal/secrets/adapters.rb index e51c73ef..70dfb2b2 100644 --- a/lib/kamal/secrets/adapters.rb +++ b/lib/kamal/secrets/adapters.rb @@ -4,6 +4,7 @@ module Kamal::Secrets::Adapters name = "one_password" if name.downcase == "1password" name = "last_pass" if name.downcase == "lastpass" name = "gcp_secret_manager" if name.downcase == "gcp" + name = "bitwarden_secrets_manager" if name.downcase == "bitwarden-sm" adapter_class(name) end diff --git a/lib/kamal/secrets/adapters/aws_secrets_manager.rb b/lib/kamal/secrets/adapters/aws_secrets_manager.rb index e23ea1f1..4bcac21d 100644 --- a/lib/kamal/secrets/adapters/aws_secrets_manager.rb +++ b/lib/kamal/secrets/adapters/aws_secrets_manager.rb @@ -6,20 +6,28 @@ class Kamal::Secrets::Adapters::AwsSecretsManager < Kamal::Secrets::Adapters::Ba def fetch_secrets(secrets, account:, session:) {}.tap do |results| - JSON.parse(get_from_secrets_manager(secrets, account: account))["SecretValues"].each do |secret| + get_from_secrets_manager(secrets, account: account).each do |secret| secret_name = secret["Name"] secret_string = JSON.parse(secret["SecretString"]) secret_string.each do |key, value| results["#{secret_name}/#{key}"] = value end + rescue JSON::ParserError + results["#{secret_name}"] = secret["SecretString"] end end end def get_from_secrets_manager(secrets, account:) - `aws secretsmanager batch-get-secret-value --secret-id-list #{secrets.map(&:shellescape).join(" ")} --profile #{account.shellescape}`.tap do - raise RuntimeError, "Could not read #{secret} from AWS Secrets Manager" unless $?.success? + `aws secretsmanager batch-get-secret-value --secret-id-list #{secrets.map(&:shellescape).join(" ")} --profile #{account.shellescape}`.tap do |secrets| + raise RuntimeError, "Could not read #{secrets} from AWS Secrets Manager" unless $?.success? + + secrets = JSON.parse(secrets) + + return secrets["SecretValues"] unless secrets["Errors"].present? + + raise RuntimeError, secrets["Errors"].map { |error| "#{error['SecretId']}: #{error['Message']}" }.join(" ") end end diff --git a/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb b/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb new file mode 100644 index 00000000..f0a19caa --- /dev/null +++ b/lib/kamal/secrets/adapters/bitwarden_secrets_manager.rb @@ -0,0 +1,67 @@ +class Kamal::Secrets::Adapters::BitwardenSecretsManager < Kamal::Secrets::Adapters::Base + def requires_account? + false + end + + private + LIST_ALL_SELECTOR = "all" + LIST_ALL_FROM_PROJECT_SUFFIX = "/all" + LIST_COMMAND = "secret list -o env" + GET_COMMAND = "secret get -o env" + + def fetch_secrets(secrets, account:, session:) + raise RuntimeError, "You must specify what to retrieve from Bitwarden Secrets Manager" if secrets.length == 0 + + if secrets.length == 1 + if secrets[0] == LIST_ALL_SELECTOR + command = LIST_COMMAND + elsif secrets[0].end_with?(LIST_ALL_FROM_PROJECT_SUFFIX) + project = secrets[0].split(LIST_ALL_FROM_PROJECT_SUFFIX).first + command = "#{LIST_COMMAND} #{project}" + end + end + + {}.tap do |results| + if command.nil? + secrets.each do |secret_uuid| + secret = run_command("#{GET_COMMAND} #{secret_uuid}") + raise RuntimeError, "Could not read #{secret_uuid} from Bitwarden Secrets Manager" unless $?.success? + key, value = parse_secret(secret) + results[key] = value + end + else + secrets = run_command(command) + raise RuntimeError, "Could not read secrets from Bitwarden Secrets Manager" unless $?.success? + secrets.split("\n").each do |secret| + key, value = parse_secret(secret) + results[key] = value + end + end + end + end + + def parse_secret(secret) + key, value = secret.split("=", 2) + value = value.gsub(/^"|"$/, "") + [ key, value ] + end + + def run_command(command, session: nil) + full_command = [ "bws", command ].join(" ") + `#{full_command}` + end + + def login(account) + run_command("run 'echo OK'") + raise RuntimeError, "Could not authenticate to Bitwarden Secrets Manager. Did you set a valid access token?" unless $?.success? + end + + def check_dependencies! + raise RuntimeError, "Bitwarden Secrets Manager CLI is not installed" unless cli_installed? + end + + def cli_installed? + `bws --version 2> /dev/null` + $?.success? + end +end diff --git a/lib/kamal/version.rb b/lib/kamal/version.rb index 2f9a86b6..9f3a9126 100644 --- a/lib/kamal/version.rb +++ b/lib/kamal/version.rb @@ -1,3 +1,3 @@ module Kamal - VERSION = "2.3.0" + VERSION = "2.4.0" end diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 2e532730..fa5049ba 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -73,7 +73,7 @@ class CliAppTest < CliTestCase run_command("boot", config: :with_assets).tap do |output| assert_match "docker tag dhh/app:latest dhh/app:latest", output assert_match "/usr/bin/env mkdir -p .kamal/apps/app/assets/volumes/web-latest ; cp -rnT .kamal/apps/app/assets/extracted/web-latest .kamal/apps/app/assets/volumes/web-latest ; cp -rnT .kamal/apps/app/assets/extracted/web-latest .kamal/apps/app/assets/volumes/web-123 || true ; cp -rnT .kamal/apps/app/assets/extracted/web-123 .kamal/apps/app/assets/volumes/web-latest || true", output - assert_match "/usr/bin/env mkdir -p .kamal/apps/app/assets/extracted/web-latest && docker stop -t 1 app-web-assets 2> /dev/null || true && docker run --name app-web-assets --detach --rm --entrypoint sleep dhh/app:latest 1000000 && docker cp -L app-web-assets:/public/assets/. .kamal/apps/app/assets/extracted/web-latest && docker stop -t 1 app-web-assets", output + assert_match "/usr/bin/env mkdir -p .kamal/apps/app/assets/extracted/web-latest && docker container rm app-web-assets 2> /dev/null || true && docker container create --name app-web-assets dhh/app:latest && docker container cp -L app-web-assets:/public/assets/. .kamal/apps/app/assets/extracted/web-latest && docker container rm app-web-assets", output assert_match /docker run --detach --restart unless-stopped --name app-web-latest --network kamal --hostname 1.1.1.1-[0-9a-f]{12} /, output assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output assert_match "/usr/bin/env find .kamal/apps/app/assets/extracted -maxdepth 1 -name 'web-*' ! -name web-latest -exec rm -rf \"{}\" + ; find .kamal/apps/app/assets/volumes -maxdepth 1 -name 'web-*' ! -name web-latest -exec rm -rf \"{}\" +", output @@ -382,8 +382,10 @@ class CliAppTest < CliTestCase test "version through main" do - stdouted { Kamal::Cli::Main.start([ "app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1" ]) }.tap do |output| - assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output + with_argv([ "app", "version", "-c", "test/fixtures/deploy_with_accessories.yml", "--hosts", "1.1.1.1" ]) do + stdouted { Kamal::Cli::Main.start }.tap do |output| + assert_match "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", output + end end end diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb index 4259fa5b..88f00743 100644 --- a/test/cli/build_test.rb +++ b/test/cli/build_test.rb @@ -274,17 +274,4 @@ class CliBuildTest < CliTestCase SSHKit::Backend::Abstract.any_instance.stubs(:execute) .with { |*args| args[0..1] == [ :docker, :buildx ] } end - - def with_build_directory - build_directory = File.join Dir.tmpdir, "kamal-clones", "app-#{pwd_sha}", "kamal" - FileUtils.mkdir_p build_directory - FileUtils.touch File.join build_directory, "Dockerfile" - yield build_directory + "/" - ensure - FileUtils.rm_rf build_directory - end - - def pwd_sha - Digest::SHA256.hexdigest(Dir.pwd)[0..12] - end end diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb index 27bf7b69..d4b57923 100644 --- a/test/cli/cli_test_case.rb +++ b/test/cli/cli_test_case.rb @@ -51,4 +51,17 @@ class CliTestCase < ActiveSupport::TestCase ensure ARGV.replace(old_argv) end + + def with_build_directory + build_directory = File.join Dir.tmpdir, "kamal-clones", "app-#{pwd_sha}", "kamal" + FileUtils.mkdir_p build_directory + FileUtils.touch File.join build_directory, "Dockerfile" + yield build_directory + "/" + ensure + FileUtils.rm_rf build_directory + end + + def pwd_sha + Digest::SHA256.hexdigest(Dir.pwd)[0..12] + end end diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index cd0efe1f..8ecb1cfa 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -460,6 +460,7 @@ class CliMainTest < CliTestCase test "run an alias for a console" do run_command("console", config_file: "deploy_with_aliases").tap do |output| + assert_no_match "App Host: 1.1.1.4", output assert_match "docker exec app-console-999 bin/console on 1.1.1.5", output assert_match "App Host: 1.1.1.5", output end @@ -486,6 +487,33 @@ class CliMainTest < CliTestCase end end + test "switch config file with an alias" do + with_config_files do + with_argv([ "other_config" ]) do + stdouted { Kamal::Cli::Main.start }.tap do |output| + assert_match ":service_with_version: app2-999", output + end + end + end + end + + test "switch destination with an alias" do + with_config_files do + with_argv([ "other_destination_config" ]) do + stdouted { Kamal::Cli::Main.start }.tap do |output| + assert_match ":service_with_version: app3-999", output + end + end + end + end + + test "run on primary via alias" do + run_command("primary_details", config_file: "deploy_with_aliases").tap do |output| + assert_match "App Host: 1.1.1.1", output + assert_no_match "App Host: 1.1.1.2", output + end + end + test "upgrade" do invoke_options = { "config_file" => "test/fixtures/deploy_with_accessories.yml", "skip_hooks" => false, "confirmed" => true, "rolling" => false } Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:proxy:upgrade", [], invoke_options) @@ -530,6 +558,20 @@ class CliMainTest < CliTestCase end end + def with_config_files + Dir.mktmpdir do |tmpdir| + config_dir = File.join(tmpdir, "config") + FileUtils.mkdir_p(config_dir) + FileUtils.cp "test/fixtures/deploy.yml", config_dir + FileUtils.cp "test/fixtures/deploy2.yml", config_dir + FileUtils.cp "test/fixtures/deploy.elsewhere.yml", config_dir + + Dir.chdir(tmpdir) do + yield + end + end + end + def assert_file(file, content) assert_match content, File.read(file) end diff --git a/test/cli/proxy_test.rb b/test/cli/proxy_test.rb index 0a890451..381fee55 100644 --- a/test/cli/proxy_test.rb +++ b/test/cli/proxy_test.rb @@ -281,6 +281,32 @@ class CliProxyTest < CliTestCase end end + test "boot_config set bind IP" do + run_command("boot_config", "set", "--publish-host-ip", "127.0.0.1").tap do |output| + %w[ 1.1.1.1 1.1.1.2 ].each do |host| + assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output + assert_match "Uploading \"--publish 127.0.0.1:80:80 --publish 127.0.0.1:443:443 --log-opt max-size=10m\" to .kamal/proxy/options on #{host}", output + end + end + end + + test "boot_config set multiple bind IPs" do + run_command("boot_config", "set", "--publish-host-ip", "127.0.0.1", "--publish-host-ip", "::1").tap do |output| + %w[ 1.1.1.1 1.1.1.2 ].each do |host| + assert_match "Running /usr/bin/env mkdir -p .kamal/proxy on #{host}", output + assert_match "Uploading \"--publish 127.0.0.1:80:80 --publish 127.0.0.1:443:443 --publish [::1]:80:80 --publish [::1]:443:443 --log-opt max-size=10m\" to .kamal/proxy/options on #{host}", output + end + end + end + + test "boot_config set invalid bind IPs" do + exception = assert_raises do + run_command("boot_config", "set", "--publish-host-ip", "1.2.3.invalidIP", "--publish-host-ip", "::1") + end + + assert_includes exception.message, "Invalid publish IP address: 1.2.3.invalidIP" + end + test "boot_config set docker options" do run_command("boot_config", "set", "--docker_options", "label=foo=bar", "add_host=thishost:thathost").tap do |output| %w[ 1.1.1.1 1.1.1.2 ].each do |host| diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index d53b31e1..75241597 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -469,10 +469,10 @@ class CommandsAppTest < ActiveSupport::TestCase test "extract assets" do assert_equal [ :mkdir, "-p", ".kamal/apps/app/assets/extracted/web-999", "&&", - :docker, :stop, "-t 1", "app-web-assets", "2> /dev/null", "|| true", "&&", - :docker, :run, "--name", "app-web-assets", "--detach", "--rm", "--entrypoint", "sleep", "dhh/app:999", "1000000", "&&", - :docker, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/apps/app/assets/extracted/web-999", "&&", - :docker, :stop, "-t 1", "app-web-assets" + :docker, :container, :rm, "app-web-assets", "2> /dev/null", "|| true", "&&", + :docker, :container, :create, "--name", "app-web-assets", "dhh/app:999", "&&", + :docker, :container, :cp, "-L", "app-web-assets:/public/assets/.", ".kamal/apps/app/assets/extracted/web-999", "&&", + :docker, :container, :rm, "app-web-assets" ], new_command(asset_path: "/public/assets").extract_assets end diff --git a/test/fixtures/deploy.elsewhere.yml b/test/fixtures/deploy.elsewhere.yml new file mode 100644 index 00000000..479e0f9d --- /dev/null +++ b/test/fixtures/deploy.elsewhere.yml @@ -0,0 +1,12 @@ +service: app3 +image: dhh/app3 +servers: + - "1.1.1.3" + - "1.1.1.4" +registry: + username: user + password: pw +builder: + arch: amd64 +aliases: + other_config: config -c config/deploy2.yml diff --git a/test/fixtures/deploy.yml b/test/fixtures/deploy.yml new file mode 100644 index 00000000..c532000f --- /dev/null +++ b/test/fixtures/deploy.yml @@ -0,0 +1,13 @@ +service: app +image: dhh/app +servers: + - "1.1.1.1" + - "1.1.1.2" +registry: + username: user + password: pw +builder: + arch: amd64 +aliases: + other_config: config -c config/deploy2.yml + other_destination_config: config -d elsewhere diff --git a/test/fixtures/deploy2.yml b/test/fixtures/deploy2.yml new file mode 100644 index 00000000..14b56951 --- /dev/null +++ b/test/fixtures/deploy2.yml @@ -0,0 +1,12 @@ +service: app2 +image: dhh/app2 +servers: + - "1.1.1.1" + - "1.1.1.2" +registry: + username: user2 + password: pw2 +builder: + arch: amd64 +aliases: + other_config: config -c config/deploy2.yml diff --git a/test/fixtures/deploy_with_aliases.yml b/test/fixtures/deploy_with_aliases.yml index ec7b14a0..104fc462 100644 --- a/test/fixtures/deploy_with_aliases.yml +++ b/test/fixtures/deploy_with_aliases.yml @@ -21,3 +21,6 @@ aliases: console: app exec --reuse -p -r console "bin/console" exec: app exec --reuse -p -r console rails: app exec --reuse -p -r console rails + primary_details: details -p + deploy_secondary: deploy -d secondary + diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb index ce32e640..a48051fe 100644 --- a/test/integration/main_test.rb +++ b/test/integration/main_test.rb @@ -90,9 +90,9 @@ class MainTest < IntegrationTest test "setup and remove" do @app = "app_with_roles" - kamal :proxy, :set_config, + kamal :proxy, :boot_config, "set", "--publish=false", - "--options=label=traefik.http.services.kamal_proxy.loadbalancer.server.scheme=http", + "--docker-options=label=traefik.http.services.kamal_proxy.loadbalancer.server.scheme=http", "label=traefik.http.routers.kamal_proxy.rule=PathPrefix\\\(\\\`/\\\`\\\)", "label=traefik.http.routers.kamal_proxy.priority=2" diff --git a/test/secrets/aws_secrets_manager_adapter_test.rb b/test/secrets/aws_secrets_manager_adapter_test.rb index 42a0f48a..7616342d 100644 --- a/test/secrets/aws_secrets_manager_adapter_test.rb +++ b/test/secrets/aws_secrets_manager_adapter_test.rb @@ -1,6 +1,35 @@ require "test_helper" class AwsSecretsManagerAdapterTest < SecretAdapterTestCase + test "fails when errors are present" do + stub_ticks.with("aws --version 2> /dev/null") + stub_ticks + .with("aws secretsmanager batch-get-secret-value --secret-id-list unknown1 unknown2 --profile default") + .returns(<<~JSON) + { + "SecretValues": [], + "Errors": [ + { + "SecretId": "unknown1", + "ErrorCode": "ResourceNotFoundException", + "Message": "Secrets Manager can't find the specified secret." + }, + { + "SecretId": "unknown2", + "ErrorCode": "ResourceNotFoundException", + "Message": "Secrets Manager can't find the specified secret." + } + ] + } + JSON + + error = assert_raises RuntimeError do + JSON.parse(shellunescape(run_command("fetch", "unknown1", "unknown2"))) + end + + assert_equal [ "unknown1: Secrets Manager can't find the specified secret.", "unknown2: Secrets Manager can't find the specified secret." ].join(" "), error.message + end + test "fetch" do stub_ticks.with("aws --version 2> /dev/null") stub_ticks @@ -44,6 +73,48 @@ class AwsSecretsManagerAdapterTest < SecretAdapterTestCase assert_equal expected_json, json end + test "fetch with string value" do + stub_ticks.with("aws --version 2> /dev/null") + stub_ticks + .with("aws secretsmanager batch-get-secret-value --secret-id-list secret secret2/KEY1 --profile default") + .returns(<<~JSON) + { + "SecretValues": [ + { + "ARN": "arn:aws:secretsmanager:us-east-1:aaaaaaaaaaaa:secret:secret", + "Name": "secret", + "VersionId": "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv", + "SecretString": "a-string-secret", + "VersionStages": [ + "AWSCURRENT" + ], + "CreatedDate": "2024-01-01T00:00:00.000000" + }, + { + "ARN": "arn:aws:secretsmanager:us-east-1:aaaaaaaaaaaa:secret:secret2", + "Name": "secret2", + "VersionId": "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv", + "SecretString": "{\\"KEY2\\":\\"VALUE2\\"}", + "VersionStages": [ + "AWSCURRENT" + ], + "CreatedDate": "2024-01-01T00:00:00.000000" + } + ], + "Errors": [] + } + JSON + + json = JSON.parse(shellunescape(run_command("fetch", "secret", "secret2/KEY1"))) + + expected_json = { + "secret"=>"a-string-secret", + "secret2/KEY2"=>"VALUE2" + } + + assert_equal expected_json, json + end + test "fetch with secret names" do stub_ticks.with("aws --version 2> /dev/null") stub_ticks diff --git a/test/secrets/bitwarden_secrets_manager_adapter_test.rb b/test/secrets/bitwarden_secrets_manager_adapter_test.rb new file mode 100644 index 00000000..1723da42 --- /dev/null +++ b/test/secrets/bitwarden_secrets_manager_adapter_test.rb @@ -0,0 +1,119 @@ +require "test_helper" + +class BitwardenSecretsManagerAdapterTest < SecretAdapterTestCase + test "fetch with no parameters" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + + error = assert_raises RuntimeError do + (shellunescape(run_command("fetch"))) + end + assert_equal("You must specify what to retrieve from Bitwarden Secrets Manager", error.message) + end + + test "fetch all" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks + .with("bws secret list -o env") + .returns("KAMAL_REGISTRY_PASSWORD=\"some_password\"\nMY_OTHER_SECRET=\"my=weird\"secret\"") + + expected = '{"KAMAL_REGISTRY_PASSWORD":"some_password","MY_OTHER_SECRET":"my\=weird\"secret"}' + actual = shellunescape(run_command("fetch", "all")) + assert_equal expected, actual + end + + test "fetch all with from" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks + .with("bws secret list -o env 82aeb5bd-6958-4a89-8197-eacab758acce") + .returns("KAMAL_REGISTRY_PASSWORD=\"some_password\"\nMY_OTHER_SECRET=\"my=weird\"secret\"") + + expected = '{"KAMAL_REGISTRY_PASSWORD":"some_password","MY_OTHER_SECRET":"my\=weird\"secret"}' + actual = shellunescape(run_command("fetch", "all", "--from", "82aeb5bd-6958-4a89-8197-eacab758acce")) + assert_equal expected, actual + end + + test "fetch item" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks + .with("bws secret get -o env 82aeb5bd-6958-4a89-8197-eacab758acce") + .returns("KAMAL_REGISTRY_PASSWORD=\"some_password\"") + + expected = '{"KAMAL_REGISTRY_PASSWORD":"some_password"}' + actual = shellunescape(run_command("fetch", "82aeb5bd-6958-4a89-8197-eacab758acce")) + assert_equal expected, actual + end + + test "fetch with multiple items" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks + .with("bws secret get -o env 82aeb5bd-6958-4a89-8197-eacab758acce") + .returns("KAMAL_REGISTRY_PASSWORD=\"some_password\"") + stub_ticks + .with("bws secret get -o env 6f8cdf27-de2b-4c77-a35d-07df8050e332") + .returns("MY_OTHER_SECRET=\"my=weird\"secret\"") + + expected = '{"KAMAL_REGISTRY_PASSWORD":"some_password","MY_OTHER_SECRET":"my\=weird\"secret"}' + actual = shellunescape(run_command("fetch", "82aeb5bd-6958-4a89-8197-eacab758acce", "6f8cdf27-de2b-4c77-a35d-07df8050e332")) + assert_equal expected, actual + end + + test "fetch all empty" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks_with("bws secret list -o env", succeed: false).returns("Error:\n0: Received error message from server") + + error = assert_raises RuntimeError do + (shellunescape(run_command("fetch", "all"))) + end + assert_equal("Could not read secrets from Bitwarden Secrets Manager", error.message) + end + + test "fetch nonexistent item" do + stub_ticks.with("bws --version 2> /dev/null") + stub_login + stub_ticks_with("bws secret get -o env 82aeb5bd-6958-4a89-8197-eacab758acce", succeed: false) + .returns("ERROR (RuntimeError): Could not read 82aeb5bd-6958-4a89-8197-eacab758acce from Bitwarden Secrets Manager") + + error = assert_raises RuntimeError do + (shellunescape(run_command("fetch", "82aeb5bd-6958-4a89-8197-eacab758acce"))) + end + assert_equal("Could not read 82aeb5bd-6958-4a89-8197-eacab758acce from Bitwarden Secrets Manager", error.message) + end + + test "fetch with no access token" do + stub_ticks.with("bws --version 2> /dev/null") + stub_ticks_with("bws run 'echo OK'", succeed: false) + + error = assert_raises RuntimeError do + (shellunescape(run_command("fetch", "all"))) + end + assert_equal("Could not authenticate to Bitwarden Secrets Manager. Did you set a valid access token?", error.message) + end + + test "fetch without CLI installed" do + stub_ticks_with("bws --version 2> /dev/null", succeed: false) + + error = assert_raises RuntimeError do + shellunescape(run_command("fetch")) + end + assert_equal "Bitwarden Secrets Manager CLI is not installed", error.message + end + + private + def stub_login + stub_ticks.with("bws run 'echo OK'").returns("OK") + end + + def run_command(*command) + stdouted do + Kamal::Cli::Secrets.start \ + [ *command, + "--adapter", "bitwarden-sm" ] + end + end +end diff --git a/test/secrets_test.rb b/test/secrets_test.rb index aca9cebe..f0ca7e29 100644 --- a/test/secrets_test.rb +++ b/test/secrets_test.rb @@ -20,6 +20,20 @@ class SecretsTest < ActiveSupport::TestCase end end + test "env references" do + with_test_secrets("secrets" => "SECRET1=$SECRET1") do + ENV["SECRET1"] = "ABC" + assert_equal "ABC", Kamal::Secrets.new["SECRET1"] + end + end + + test "secrets file value overrides env" do + with_test_secrets("secrets" => "SECRET1=DEF") do + ENV["SECRET1"] = "ABC" + assert_equal "DEF", Kamal::Secrets.new["SECRET1"] + end + end + test "destinations" do with_test_secrets("secrets.dest" => "SECRET=DEF", "secrets" => "SECRET=ABC", "secrets-common" => "SECRET=GHI\nSECRET2=JKL") do assert_equal "ABC", Kamal::Secrets.new["SECRET"]