spacebar/kamal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'basecamp:main' into buildpacks

Browse Source
This commit is contained in:
Nick Hammond
2024-09-16 18:11:33 -07:00
committed by GitHub
parent 2c5f2a7ce0 66d5e25834
commit 548452aa12
85 changed files with 1539 additions and 746 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lib/kamal.rb
View File

@@ -5,8 +5,10 @@ end
require "active_support"
require "zeitwerk"
require "yaml"
require "tmpdir"
require "pathname"
loader = Zeitwerk::Loader.for_gem
loader.ignore(File.join(__dir__, "kamal", "sshkit_with_ext.rb"))
loader.setup
loader.eager_load # We need all commands loaded.
loader.eager_load_namespace(Kamal::Cli) # We need all commands loaded.

2
lib/kamal/cli/accessory.rb
View File

@@ -12,6 +12,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
on(hosts) do
execute *KAMAL.registry.login if login
execute *KAMAL.auditor.record("Booted #{name} accessory"), verbosity: :debug
execute *accessory.ensure_env_directory
upload! accessory.secrets_io, accessory.secrets_path, mode: "0600"
execute *accessory.run
end
end

14
lib/kamal/cli/app/boot.rb
View File

@@ -1,6 +1,6 @@
class Kamal::Cli::App::Boot
attr_reader :host, :role, :version, :barrier, :sshkit
delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, to: :sshkit
delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, :upload!, to: :sshkit
delegate :uses_cord?, :assets?, :running_traefik?, to: :role
def initialize(host, role, sshkit, version, barrier)
@@ -48,7 +48,11 @@ class Kamal::Cli::App::Boot
execute *app.tie_cord(role.cord_host_file) if uses_cord?
hostname = "#{host.to_s[0...51].gsub(/\.+$/, '')}-#{SecureRandom.hex(6)}"
execute *app.ensure_env_directory
upload! role.secrets_io(host), role.secrets_path, mode: "0600"
execute *app.run(hostname: hostname)
Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
end
@@ -88,8 +92,12 @@ class Kamal::Cli::App::Boot
def close_barrier
if barrier.close
info "First #{KAMAL.primary_role} container is unhealthy on #{host}, not booting any other roles"
error capture_with_info(*app.logs(version: version))
error capture_with_info(*app.container_health_log(version: version))
begin
error capture_with_info(*app.logs(version: version))
error capture_with_info(*app.container_health_log(version: version))
rescue SSHKit::Command::Failed
error "Could not fetch logs for #{version}"
end
end
end

47
lib/kamal/cli/base.rb
View File

@@ -1,5 +1,4 @@
require "thor"
require "dotenv"
require "kamal/sshkit_with_ext"
module Kamal::Cli
@@ -31,53 +30,15 @@ module Kamal::Cli
else
super
end
@original_env = ENV.to_h.dup
load_env
initialize_commander(options_with_subcommand_class_options)
initialize_commander unless KAMAL.configured?
end
private
def reload_env
reset_env
load_env
end
def load_env
if destination = options[:destination]
Dotenv.load(".env.#{destination}", ".env")
else
Dotenv.load(".env")
end
end
def reset_env
replace_env @original_env
end
def replace_env(env)
ENV.clear
ENV.update(env)
end
def with_original_env
keeping_current_env do
reset_env
yield
end
end
def keeping_current_env
current_env = ENV.to_h.dup
yield
ensure
replace_env(current_env)
end
def options_with_subcommand_class_options
options.merge(@_initializer.last[:class_options] || {})
end
def initialize_commander(options)
def initialize_commander
KAMAL.tap do |commander|
if options[:verbose]
ENV["VERBOSE"] = "1" # For backtraces via cli/start
@@ -112,8 +73,6 @@ module Kamal::Cli
if KAMAL.holding_lock?
yield
else
ensure_run_and_locks_directory
acquire_lock
begin
@@ -142,6 +101,8 @@ module Kamal::Cli
end
def acquire_lock
ensure_run_and_locks_directory
raise_if_locked do
say "Acquiring the deploy lock...", :magenta
on(KAMAL.primary_host) { execute *KAMAL.lock.acquire("Automatic deploy lock", KAMAL.config.version), verbosity: :debug }

2
lib/kamal/cli/build.rb
View File

@@ -51,7 +51,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base
push = KAMAL.builder.push
KAMAL.with_verbosity(:debug) do
Dir.chdir(KAMAL.config.builder.build_directory) { execute *push }
Dir.chdir(KAMAL.config.builder.build_directory) { execute *push, env: KAMAL.config.builder.secrets }
end
end
end

54
lib/kamal/cli/env.rb
View File

@@ -1,54 +0,0 @@
require "tempfile"
class Kamal::Cli::Env < Kamal::Cli::Base
desc "push", "Push the env file to the remote hosts"
def push
with_lock do
on(KAMAL.hosts) do
execute *KAMAL.auditor.record("Pushed env files"), verbosity: :debug
KAMAL.roles_on(host).each do |role|
execute *KAMAL.app(role: role, host: host).make_env_directory
upload! role.env(host).secrets_io, role.env(host).secrets_file, mode: 400
end
end
on(KAMAL.traefik_hosts) do
execute *KAMAL.traefik.make_env_directory
upload! KAMAL.traefik.env.secrets_io, KAMAL.traefik.env.secrets_file, mode: 400
end
on(KAMAL.accessory_hosts) do
KAMAL.accessories_on(host).each do |accessory|
accessory_config = KAMAL.config.accessory(accessory)
execute *KAMAL.accessory(accessory).make_env_directory
upload! accessory_config.env.secrets_io, accessory_config.env.secrets_file, mode: 400
end
end
end
end
desc "delete", "Delete the env file from the remote hosts"
def delete
with_lock do
on(KAMAL.hosts) do
execute *KAMAL.auditor.record("Deleted env files"), verbosity: :debug
KAMAL.roles_on(host).each do |role|
execute *KAMAL.app(role: role, host: host).remove_env_file
end
end
on(KAMAL.traefik_hosts) do
execute *KAMAL.traefik.remove_env_file
end
on(KAMAL.accessory_hosts) do
KAMAL.accessories_on(host).each do |accessory|
accessory_config = KAMAL.config.accessory(accessory)
execute *KAMAL.accessory(accessory).remove_env_file
end
end
end
end
end

2
lib/kamal/cli/healthcheck/barrier.rb
View File

@@ -1,3 +1,5 @@
require "concurrent/ivar"
class Kamal::Cli::Healthcheck::Barrier
def initialize
@ivar = Concurrent::IVar.new

5
lib/kamal/cli/lock.rb
View File

@@ -3,7 +3,6 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
def status
handle_missing_lock do
on(KAMAL.primary_host) do
execute *KAMAL.server.ensure_run_directory
puts capture_with_debug(*KAMAL.lock.status)
end
end
@@ -13,9 +12,10 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
option :message, aliases: "-m", type: :string, desc: "A lock message", required: true
def acquire
message = options[:message]
ensure_run_and_locks_directory
raise_if_locked do
on(KAMAL.primary_host) do
execute *KAMAL.server.ensure_run_directory
execute *KAMAL.lock.acquire(message, KAMAL.config.version), verbosity: :debug
end
say "Acquired the deploy lock"
@@ -26,7 +26,6 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
def release
handle_missing_lock do
on(KAMAL.primary_host) do
execute *KAMAL.server.ensure_run_directory
execute *KAMAL.lock.release, verbosity: :debug
end
say "Released the deploy lock"

54
lib/kamal/cli/main.rb
View File

@@ -9,10 +9,6 @@ class Kamal::Cli::Main < Kamal::Cli::Base
say "Ensure Docker is installed...", :magenta
invoke "kamal:cli:server:bootstrap", [], invoke_options
say "Evaluate and push env files...", :magenta
invoke "kamal:cli:main:envify", [], invoke_options
invoke "kamal:cli:env:push", [], invoke_options
invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options
deploy
end
@@ -37,7 +33,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
end
with_lock do
run_hook "pre-deploy"
run_hook "pre-deploy", secrets: true
say "Ensure Traefik is running...", :magenta
invoke "kamal:cli:traefik:boot", [], invoke_options
@@ -52,7 +48,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
end
end
run_hook "post-deploy", runtime: runtime.round
run_hook "post-deploy", secrets: true, runtime: runtime.round
end
desc "redeploy", "Deploy app to servers without bootstrapping servers, starting Traefik, pruning, and registry login"
@@ -70,7 +66,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
end
with_lock do
run_hook "pre-deploy"
run_hook "pre-deploy", secrets: true
say "Detect stale containers...", :magenta
invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
@@ -79,7 +75,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
end
end
run_hook "post-deploy", runtime: runtime.round
run_hook "post-deploy", secrets: true, runtime: runtime.round
end
desc "rollback [VERSION]", "Rollback app to VERSION"
@@ -93,7 +89,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
old_version = nil
if container_available?(version)
run_hook "pre-deploy"
run_hook "pre-deploy", secrets: true
invoke "kamal:cli:app:boot", [], invoke_options.merge(version: version)
rolled_back = true
@@ -103,7 +99,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
end
end
run_hook "post-deploy", runtime: runtime.round if rolled_back
run_hook "post-deploy", secrets: true, runtime: runtime.round if rolled_back
end
desc "details", "Show details about all containers"
@@ -152,9 +148,10 @@ class Kamal::Cli::Main < Kamal::Cli::Base
puts "Created configuration file in config/deploy.yml"
end
unless (deploy_file = Pathname.new(File.expand_path(".env"))).exist?
FileUtils.cp_r Pathname.new(File.expand_path("templates/template.env", __dir__)), deploy_file
puts "Created .env file"
unless (secrets_file = Pathname.new(File.expand_path(".kamal/secrets"))).exist?
FileUtils.mkdir_p secrets_file.dirname
FileUtils.cp_r Pathname.new(File.expand_path("templates/secrets", __dir__)), secrets_file
puts "Created .kamal/secrets file"
end
unless (hooks_dir = Pathname.new(File.expand_path(".kamal/hooks"))).exist?
@@ -179,31 +176,6 @@ class Kamal::Cli::Main < Kamal::Cli::Base
end
end
desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)"
option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip .env file push"
def envify
if destination = options[:destination]
env_template_path = ".env.#{destination}.erb"
env_path = ".env.#{destination}"
else
env_template_path = ".env.erb"
env_path = ".env"
end
if Pathname.new(File.expand_path(env_template_path)).exist?
# Ensure existing env doesn't pollute template evaluation
content = with_original_env { ERB.new(File.read(env_template_path), trim_mode: "-").result }
File.write(env_path, content, perm: 0600)
unless options[:skip_push]
reload_env
invoke "kamal:cli:env:push", options
end
else
puts "Skipping envify (no #{env_template_path} exist)"
end
end
desc "remove", "Remove Traefik, app, accessories, and registry session from servers"
option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
def remove
@@ -231,9 +203,6 @@ class Kamal::Cli::Main < Kamal::Cli::Base
desc "build", "Build application image"
subcommand "build", Kamal::Cli::Build
desc "env", "Manage environment files"
subcommand "env", Kamal::Cli::Env
desc "lock", "Manage the deploy lock"
subcommand "lock", Kamal::Cli::Lock
@@ -243,6 +212,9 @@ class Kamal::Cli::Main < Kamal::Cli::Base
desc "registry", "Login and -out of the image registry"
subcommand "registry", Kamal::Cli::Registry
desc "secrets", "Helpers for extracting secrets"
subcommand "secrets", Kamal::Cli::Secrets
desc "server", "Bootstrap servers with curl and Docker"
subcommand "server", Kamal::Cli::Server

36
lib/kamal/cli/secrets.rb Normal file
View File

@@ -0,0 +1,36 @@
class Kamal::Cli::Secrets < Kamal::Cli::Base
desc "fetch [SECRETS...]", "Fetch secrets from a vault"
option :adapter, type: :string, aliases: "-a", required: true, desc: "Which vault adapter to use"
option :account, type: :string, required: true, desc: "The account identifier or username"
option :from, type: :string, required: false, desc: "A vault or folder to fetch the secrets from"
option :inline, type: :boolean, required: false, hidden: true
def fetch(*secrets)
results = adapter(options[:adapter]).fetch(secrets, **options.slice(:account, :from).symbolize_keys)
return_or_puts JSON.dump(results).shellescape, inline: options[:inline]
end
desc "extract", "Extract a single secret from the results of a fetch call"
option :inline, type: :boolean, required: false, hidden: true
def extract(name, secrets)
parsed_secrets = JSON.parse(secrets)
value = parsed_secrets[name] || parsed_secrets.find { |k, v| k.end_with?("/#{name}") }&.last
raise "Could not find secret #{name}" if value.nil?
return_or_puts value, inline: options[:inline]
end
private
def adapter(adapter)
Kamal::Secrets::Adapters.lookup(adapter)
end
def return_or_puts(value, inline: nil)
if inline
value
else
puts value
end
end
end

16
lib/kamal/cli/templates/secrets Normal file
View File

@@ -0,0 +1,16 @@
# WARNING: Avoid adding secrets directly to this file
# If you must, then add `.kamal/secrets*` to your .gitignore file
# Option 1: Read secrets from the environment
KAMAL_REGISTRY_PASSWORD=$KAMAL_REGISTRY_PASSWORD
# Option 2: Read secrets via a command
# RAILS_MASTER_KEY=$(cat config/master.key)
# Option 3: Read secrets via kamal secrets helpers
# These will handle logging in and fetching the secrets in as few calls as possible
# There are adapters for 1Password, LastPass + Bitwarden
#
# SECRETS=$(kamal secrets fetch --adapter 1password --account my-account --from MyVault/MyItem KAMAL_REGISTRY_PASSWORD RAILS_MASTER_KEY)
# KAMAL_REGISTRY_PASSWORD=$(kamal secrets extract KAMAL_REGISTRY_PASSWORD $SECRETS)
# RAILS_MASTER_KEY=$(kamal secrets extract RAILS_MASTER_KEY $SECRETS)

2
lib/kamal/cli/templates/template.env
View File

@@ -1,2 +0,0 @@
KAMAL_REGISTRY_PASSWORD=change-this
RAILS_MASTER_KEY=another-env

2
lib/kamal/cli/traefik.rb
View File

@@ -4,6 +4,8 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
with_lock do
on(KAMAL.traefik_hosts) do
execute *KAMAL.registry.login
execute *KAMAL.traefik.ensure_env_directory
upload! KAMAL.traefik.secrets_io, KAMAL.traefik.secrets_path, mode: "0600"
execute *KAMAL.traefik.start_or_run
end
end

5
lib/kamal/commander.rb
View File

@@ -1,5 +1,6 @@
require "active_support/core_ext/enumerable"
require "active_support/core_ext/module/delegation"
require "active_support/core_ext/object/blank"
class Kamal::Commander
attr_accessor :verbosity, :holding_lock, :connected
@@ -23,6 +24,10 @@ class Kamal::Commander
@config, @config_kwargs = nil, kwargs
end
def configured?
@config || @config_kwargs
end
attr_reader :specific_roles, :specific_hosts
def specific_primary!

12
lib/kamal/commands/accessory.rb
View File

@@ -1,7 +1,9 @@
class Kamal::Commands::Accessory < Kamal::Commands::Base
attr_reader :accessory_config
delegate :service_name, :image, :hosts, :port, :files, :directories, :cmd,
:publish_args, :env_args, :volume_args, :label_args, :option_args, to: :accessory_config
:publish_args, :env_args, :volume_args, :label_args, :option_args,
:secrets_io, :secrets_path, :env_directory,
to: :accessory_config
def initialize(config, name:)
super(config)
@@ -98,12 +100,8 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
docker :image, :rm, "--force", image
end
def make_env_directory
make_directory accessory_config.env.secrets_directory
end
def remove_env_file
[ :rm, "-f", accessory_config.env.secrets_file ]
def ensure_env_directory
make_directory env_directory
end
private

10
lib/kamal/commands/app.rb
View File

@@ -69,16 +69,10 @@ class Kamal::Commands::App < Kamal::Commands::Base
extract_version_from_name
end
def make_env_directory
make_directory role.env(host).secrets_directory
def ensure_env_directory
make_directory role.env_directory
end
def remove_env_file
[ :rm, "-f", role.env(host).secrets_file ]
end
private
def container_name(version = nil)
[ role.container_prefix, version || config.version ].compact.join("-")

2
lib/kamal/commands/app/assets.rb
View File

@@ -5,7 +5,7 @@ module Kamal::Commands::App::Assets
combine \
make_directory(role.asset_extracted_path),
[ *docker(:stop, "-t 1", asset_container, "2> /dev/null"), "|| true" ],
docker(:run, "--name", asset_container, "--detach", "--rm", config.absolute_image, "sleep 1000000"),
docker(:run, "--name", asset_container, "--detach", "--rm", "--entrypoint", "sleep", config.absolute_image, "1000000"),
docker(:cp, "-L", "#{asset_container}:#{role.asset_path}/.", role.asset_extracted_path),
docker(:stop, "-t 1", asset_container),
by: "&&"

9
lib/kamal/commands/auditor.rb
View File

@@ -8,9 +8,12 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
# Runs remotely
def record(line, **details)
append \
[ :echo, audit_tags(**details).except(:version, :service_version, :service).to_s, line ],
audit_log_file
combine \
[ :mkdir, "-p", config.run_directory ],
append(
[ :echo, audit_tags(**details).except(:version, :service_version, :service).to_s, line ],
audit_log_file
)
end
def reveal

4
lib/kamal/commands/base.rb
View File

@@ -37,6 +37,10 @@ module Kamal::Commands
[ :rm, "-r", path ]
end
def remove_file(path)
[ :rm, path ]
end
private
def combine(*commands, by: "&&")
commands

2
lib/kamal/commands/builder/base.rb
View File

@@ -79,7 +79,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
end
def build_secrets
argumentize "--secret", secrets.collect { |secret| [ "id", secret ] }
argumentize "--secret", secrets.keys.collect { |secret| [ "id", secret ] }
end
def build_dockerfile

7
lib/kamal/commands/hook.rb
View File

@@ -1,6 +1,9 @@
class Kamal::Commands::Hook < Kamal::Commands::Base
def run(hook, **details)
[ hook_file(hook), env: tags(**details).env ]
def run(hook, secrets: false, **details)
env = tags(**details).env
env.merge!(config.secrets.to_h) if secrets
[ hook_file(hook), env: env ]
end
def hook_exists?(hook)

14
lib/kamal/commands/traefik.rb
View File

@@ -1,6 +1,6 @@
class Kamal::Commands::Traefik < Kamal::Commands::Base
delegate :argumentize, :optionize, to: Kamal::Utils
delegate :port, :publish?, :labels, :env, :image, :options, :args, to: :"config.traefik"
delegate :port, :publish?, :labels, :env, :image, :options, :args, :env_args, :secrets_io, :env_directory, :secrets_path, to: :"config.traefik"
def run
docker :run, "--name traefik",
@@ -54,12 +54,8 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
end
def make_env_directory
make_directory(env.secrets_directory)
end
def remove_env_file
[ :rm, "-f", env.secrets_file ]
def ensure_env_directory
make_directory env_directory
end
private
@@ -71,10 +67,6 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
argumentize "--label", labels
end
def env_args
env.args
end
def docker_options_args
optionize(options)
end

15
lib/kamal/configuration.rb
View File

@@ -2,7 +2,6 @@ require "active_support/ordered_options"
require "active_support/core_ext/string/inquiry"
require "active_support/core_ext/module/delegation"
require "active_support/core_ext/hash/keys"
require "pathname"
require "erb"
require "net/ssh/proxy/jump"
@@ -10,7 +9,7 @@ class Kamal::Configuration
delegate :service, :image, :labels, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
delegate :argumentize, :optionize, to: Kamal::Utils
attr_reader :destination, :raw_config
attr_reader :destination, :raw_config, :secrets
attr_reader :accessories, :aliases, :boot, :builder, :env, :healthcheck, :logging, :traefik, :servers, :ssh, :sshkit, :registry
include Validation
@@ -57,7 +56,7 @@ class Kamal::Configuration
@aliases = @raw_config.aliases&.keys&.to_h { |name| [ name, Alias.new(name, config: self) ] } || {}
@boot = Boot.new(config: self)
@builder = Builder.new(config: self)
@env = Env.new(config: @raw_config.env || {})
@env = Env.new(config: @raw_config.env || {}, secrets: secrets)
@healthcheck = Healthcheck.new(healthcheck_config: @raw_config.healthcheck)
@logging = Logging.new(logging_config: @raw_config.logging)
@@ -65,6 +64,8 @@ class Kamal::Configuration
@ssh = Ssh.new(config: self)
@sshkit = Sshkit.new(config: self)
@secrets = Kamal::Secrets.new(destination: destination)
ensure_destination_if_required
ensure_required_keys_present
ensure_valid_kamal_version
@@ -218,13 +219,13 @@ class Kamal::Configuration
end
def host_env_directory
def env_directory
File.join(run_directory, "env")
end
def env_tags
@env_tags ||= if (tags = raw_config.env["tags"])
tags.collect { |name, config| Env::Tag.new(name, config: config) }
tags.collect { |name, config| Env::Tag.new(name, config: config, secrets: secrets) }
else
[]
end
@@ -254,6 +255,10 @@ class Kamal::Configuration
}.compact
end
def secrets
@secrets ||= Kamal::Secrets.new(destination: destination)
end
private
# Will raise ArgumentError if any required config keys are missing
def ensure_destination_if_required

16
lib/kamal/configuration/accessory.rb
View File

@@ -16,7 +16,7 @@ class Kamal::Configuration::Accessory
@env = Kamal::Configuration::Env.new \
config: accessory_config.fetch("env", {}),
secrets_file: File.join(config.host_env_directory, "accessories", "#{service_name}.env"),
secrets: config.secrets,
context: "accessories/#{name}/env"
end
@@ -51,7 +51,19 @@ class Kamal::Configuration::Accessory
end
def env_args
env.args
[ *env.clear_args, *argumentize("--env-file", secrets_path) ]
end
def env_directory
File.join(config.env_directory, "accessories")
end
def secrets_io
env.secrets_io
end
def secrets_path
File.join(config.env_directory, "accessories", "#{service_name}.env")
end
def files

2
lib/kamal/configuration/builder.rb
View File

@@ -66,7 +66,7 @@ class Kamal::Configuration::Builder
end
def secrets
builder_config["secrets"] || []
(builder_config["secrets"] || []).to_h { |key| [ key, config.secrets[key] ] }
end
def dockerfile

27
lib/kamal/configuration/env.rb
View File

@@ -1,36 +1,29 @@
class Kamal::Configuration::Env
include Kamal::Configuration::Validation
attr_reader :secrets_keys, :clear, :secrets_file, :context
attr_reader :context, :secrets
attr_reader :clear, :secret_keys
delegate :argumentize, to: Kamal::Utils
def initialize(config:, secrets_file: nil, context: "env")
def initialize(config:, secrets:, context: "env")
@clear = config.fetch("clear", config.key?("secret") || config.key?("tags") ? {} : config)
@secrets_keys = config.fetch("secret", [])
@secrets_file = secrets_file
@secrets = secrets
@secret_keys = config.fetch("secret", [])
@context = context
validate! config, context: context, with: Kamal::Configuration::Validator::Env
end
def args
[ "--env-file", secrets_file, *argumentize("--env", clear) ]
def clear_args
argumentize("--env", clear)
end
def secrets_io
StringIO.new(Kamal::EnvFile.new(secrets).to_s)
end
def secrets
@secrets ||= secrets_keys.to_h { |key| [ key, ENV.fetch(key) ] }
end
def secrets_directory
File.dirname(secrets_file)
Kamal::EnvFile.new(secret_keys.to_h { |key| [ key, secrets[key] ] }).to_io
end
def merge(other)
self.class.new \
config: { "clear" => clear.merge(other.clear), "secret" => secrets_keys | other.secrets_keys },
secrets_file: secrets_file || other.secrets_file
config: { "clear" => clear.merge(other.clear), "secret" => secret_keys | other.secret_keys },
secrets: secrets
end
end

7
lib/kamal/configuration/env/tag.rb vendored
View File

@@ -1,12 +1,13 @@
class Kamal::Configuration::Env::Tag
attr_reader :name, :config
attr_reader :name, :config, :secrets
def initialize(name, config:)
def initialize(name, config:, secrets:)
@name = name
@config = config
@secrets = secrets
end
def env
Kamal::Configuration::Env.new(config: config)
Kamal::Configuration::Env.new(config: config, secrets: secrets)
end
end

5
lib/kamal/configuration/registry.rb
View File

@@ -1,10 +1,11 @@
class Kamal::Configuration::Registry
include Kamal::Configuration::Validation
attr_reader :registry_config
attr_reader :registry_config, :secrets
def initialize(config:)
@registry_config = config.raw_config.registry || {}
@secrets = config.secrets
validate! registry_config, with: Kamal::Configuration::Validator::Registry
end
@@ -23,7 +24,7 @@ class Kamal::Configuration::Registry
private
def lookup(key)
if registry_config[key].is_a?(Array)
ENV.fetch(registry_config[key].first).dup
secrets[registry_config[key].first]
else
registry_config[key]
end

16
lib/kamal/configuration/role.rb
View File

@@ -18,7 +18,7 @@ class Kamal::Configuration::Role
@specialized_env = Kamal::Configuration::Env.new \
config: specializations.fetch("env", {}),
secrets_file: File.join(config.host_env_directory, "roles", "#{container_prefix}.env"),
secrets: config.secrets,
context: "servers/#{name}/env"
@specialized_logging = Kamal::Configuration::Logging.new \
@@ -77,7 +77,19 @@ class Kamal::Configuration::Role
end
def env_args(host)
env(host).args
[ *env(host).clear_args, *argumentize("--env-file", secrets_path) ]
end
def env_directory
File.join(config.env_directory, "roles")
end
def secrets_io(host)
env(host).secrets_io
end
def secrets_path
File.join(config.env_directory, "roles", "#{container_prefix}.env")
end
def asset_volume_args

20
lib/kamal/configuration/traefik.rb
View File

@@ -1,4 +1,6 @@
class Kamal::Configuration::Traefik
delegate :argumentize, to: Kamal::Utils
DEFAULT_IMAGE = "traefik:v2.10"
CONTAINER_PORT = 80
DEFAULT_ARGS = {
@@ -34,7 +36,7 @@ class Kamal::Configuration::Traefik
def env
Kamal::Configuration::Env.new \
config: traefik_config.fetch("env", {}),
secrets_file: File.join(config.host_env_directory, "traefik", "traefik.env"),
secrets: config.secrets,
context: "traefik/env"
end
@@ -57,4 +59,20 @@ class Kamal::Configuration::Traefik
def image
traefik_config.fetch("image", DEFAULT_IMAGE)
end
def env_args
[ *env.clear_args, *argumentize("--env-file", secrets_path) ]
end
def env_directory
File.join(config.env_directory, "traefik")
end
def secrets_io
env.secrets_io
end
def secrets_path
File.join(config.env_directory, "traefik", "traefik.env")
end
end

4
lib/kamal/env_file.rb
View File

@@ -15,6 +15,10 @@ class Kamal::EnvFile
env_file.presence || "\n"
end
def to_io
StringIO.new(to_s)
end
alias to_str to_s
private

37
lib/kamal/secrets.rb Normal file
View File

@@ -0,0 +1,37 @@
require "dotenv"
class Kamal::Secrets
attr_reader :secrets_files
Kamal::Secrets::Dotenv::InlineCommandSubstitution.install!
def initialize(destination: nil)
@secrets_files = \
[ ".kamal/secrets-common", ".kamal/secrets#{(".#{destination}" if destination)}" ].select { |f| File.exist?(f) }
@mutex = Mutex.new
end
def [](key)
# Fetching secrets may ask the user for input, so ensure only one thread does that
@mutex.synchronize do
secrets.fetch(key)
end
rescue KeyError
if secrets_files
raise Kamal::ConfigurationError, "Secret '#{key}' not found in #{secrets_files.join(", ")}"
else
raise Kamal::ConfigurationError, "Secret '#{key}' not found, no secret files provided"
end
end
def to_h
secrets
end
private
def secrets
@secrets ||= secrets_files.inject({}) do |secrets, secrets_file|
secrets.merge!(::Dotenv.parse(secrets_file))
end
end
end

14
lib/kamal/secrets/adapters.rb Normal file
View File

@@ -0,0 +1,14 @@
require "active_support/core_ext/string/inflections"
module Kamal::Secrets::Adapters
def self.lookup(name)
name = "one_password" if name.downcase == "1password"
name = "last_pass" if name.downcase == "lastpass"
adapter_class(name)
end
def self.adapter_class(name)
Object.const_get("Kamal::Secrets::Adapters::#{name.camelize}").new
rescue NameError => e
raise RuntimeError, "Unknown secrets adapter: #{name}"
end
end

18
lib/kamal/secrets/adapters/base.rb Normal file
View File

@@ -0,0 +1,18 @@
class Kamal::Secrets::Adapters::Base
delegate :optionize, to: Kamal::Utils
def fetch(secrets, account:, from: nil)
session = login(account)
full_secrets = secrets.map { |secret| [ from, secret ].compact.join("/") }
fetch_secrets(full_secrets, account: account, session: session)
end
private
def login(...)
raise NotImplementedError
end
def fetch_secrets(...)
raise NotImplementedError
end
end

64
lib/kamal/secrets/adapters/bitwarden.rb Normal file
View File

@@ -0,0 +1,64 @@
class Kamal::Secrets::Adapters::Bitwarden < Kamal::Secrets::Adapters::Base
private
def login(account)
status = run_command("status")
if status["status"] == "unauthenticated"
run_command("login #{account.shellescape}", raw: true)
status = run_command("status")
end
if status["status"] == "locked"
session = run_command("unlock --raw", raw: true).presence
status = run_command("status", session: session)
end
raise RuntimeError, "Failed to login to and unlock Bitwarden" unless status["status"] == "unlocked"
run_command("sync", session: session, raw: true)
raise RuntimeError, "Failed to sync Bitwarden" unless $?.success?
session
end
def fetch_secrets(secrets, account:, session:)
{}.tap do |results|
items_fields(secrets).each do |item, fields|
item_json = run_command("get item #{item.shellescape}", session: session, raw: true)
raise RuntimeError, "Could not read #{secret} from Bitwarden" unless $?.success?
item_json = JSON.parse(item_json)
if fields.any?
fields.each do |field|
item_field = item_json["fields"].find { |f| f["name"] == field }
raise RuntimeError, "Could not find field #{field} in item #{item} in Bitwarden" unless item_field
value = item_field["value"]
results["#{item}/#{field}"] = value
end
else
results[item] = item_json["login"]["password"]
end
end
end
end
def items_fields(secrets)
{}.tap do |items|
secrets.each do |secret|
item, field = secret.split("/")
items[item] ||= []
items[item] << field
end
end
end
def signedin?(account)
run_command("status")["status"] != "unauthenticated"
end
def run_command(command, session: nil, raw: false)
full_command = [ *("BW_SESSION=#{session.shellescape}" if session), "bw", command ].join(" ")
result = `#{full_command}`.strip
raw ? result : JSON.parse(result)
end
end

30
lib/kamal/secrets/adapters/last_pass.rb Normal file
View File

@@ -0,0 +1,30 @@
class Kamal::Secrets::Adapters::LastPass < Kamal::Secrets::Adapters::Base
private
def login(account)
unless loggedin?(account)
`lpass login #{account.shellescape}`
raise RuntimeError, "Failed to login to 1Password" unless $?.success?
end
end
def loggedin?(account)
`lpass status --color never`.strip == "Logged in as #{account}."
end
def fetch_secrets(secrets, account:, session:)
items = `lpass show #{secrets.map(&:shellescape).join(" ")} --json`
raise RuntimeError, "Could not read #{secrets} from 1Password" unless $?.success?
items = JSON.parse(items)
{}.tap do |results|
items.each do |item|
results[item["fullname"]] = item["password"]
end
if (missing_items = secrets - results.keys).any?
raise RuntimeError, "Could not find #{missing_items.join(", ")} in LassPass"
end
end
end
end

61
lib/kamal/secrets/adapters/one_password.rb Normal file
View File

@@ -0,0 +1,61 @@
class Kamal::Secrets::Adapters::OnePassword < Kamal::Secrets::Adapters::Base
delegate :optionize, to: Kamal::Utils
private
def login(account)
unless loggedin?(account)
`op signin #{to_options(account: account, force: true, raw: true)}`.tap do
raise RuntimeError, "Failed to login to 1Password" unless $?.success?
end
end
end
def loggedin?(account)
`op account get --account #{account.shellescape} 2> /dev/null`
$?.success?
end
def fetch_secrets(secrets, account:, session:)
{}.tap do |results|
vaults_items_fields(secrets).map do |vault, items|
items.each do |item, fields|
fields_json = JSON.parse(op_item_get(vault, item, fields, account: account, session: session))
fields_json = [ fields_json ] if fields.one?
fields_json.each do |field_json|
# The reference is in the form `op://vault/item/field[/field]`
field = field_json["reference"].delete_prefix("op://").delete_suffix("/password")
results[field] = field_json["value"]
end
end
end
end
end
def to_options(**options)
optionize(options.compact).join(" ")
end
def vaults_items_fields(secrets)
{}.tap do |vaults|
secrets.each do |secret|
secret = secret.delete_prefix("op://")
vault, item, *fields = secret.split("/")
fields << "password" if fields.empty?
vaults[vault] ||= {}
vaults[vault][item] ||= []
vaults[vault][item] << fields.join(".")
end
end
end
def op_item_get(vault, item, fields, account:, session:)
labels = fields.map { |field| "label=#{field}" }.join(",")
options = to_options(vault: vault, fields: labels, format: "json", account: account, session: session.presence)
`op item get #{item.shellescape} #{options}`.tap do
raise RuntimeError, "Could not read #{fields.join(", ")} from #{item} in the #{vault} 1Password vault" unless $?.success?
end
end
end

10
lib/kamal/secrets/adapters/test.rb Normal file
View File

@@ -0,0 +1,10 @@
class Kamal::Secrets::Adapters::Test < Kamal::Secrets::Adapters::Base
private
def login(account)
true
end
def fetch_secrets(secrets, account:, session:)
secrets.to_h { |secret| [ secret, secret.reverse ] }
end
end

32
lib/kamal/secrets/dotenv/inline_command_substitution.rb Normal file
View File

@@ -0,0 +1,32 @@
class Kamal::Secrets::Dotenv::InlineCommandSubstitution
class << self
def install!
::Dotenv::Parser.substitutions.map! { |sub| sub == ::Dotenv::Substitutions::Command ? self : sub }
end
def call(value, _env, overwrite: false)
# Process interpolated shell commands
value.gsub(Dotenv::Substitutions::Command.singleton_class::INTERPOLATED_SHELL_COMMAND) do |*|
# Eliminate opening and closing parentheses
command = $LAST_MATCH_INFO[:cmd][1..-2]
if $LAST_MATCH_INFO[:backslash]
# Command is escaped, don't replace it.
$LAST_MATCH_INFO[0][1..]
else
if command =~ /\A\s*kamal\s*secrets\s+/
# Inline the command
inline_secrets_command(command)
else
# Execute the command and return the value
`#{command}`.chomp
end
end
end
end
def inline_secrets_command(command)
Kamal::Cli::Main.start(command.shellsplit[1..] + [ "--inline" ]).chomp
end
end
end

1
lib/kamal/sshkit_with_ext.rb
View File

@@ -3,6 +3,7 @@ require "sshkit/dsl"
require "net/scp"
require "active_support/core_ext/hash/deep_merge"
require "json"
require "concurrent/atomic/semaphore"
class SSHKit::Backend::Abstract
def capture_with_info(*args, **kwargs)

8
lib/kamal/utils.rb
View File

@@ -1,3 +1,5 @@
require "active_support/core_ext/object/try"
module Kamal::Utils
extend self
@@ -54,6 +56,12 @@ module Kamal::Utils
# Escape a value to make it safe for shell use.
def escape_shell_value(value)
value.to_s.scan(/[\x00-\x7F]+|[^\x00-\x7F]+/) \
.map { |part| part.ascii_only? ? escape_ascii_shell_value(part) : part }
.join
end
def escape_ascii_shell_value(value)
value.to_s.dump
.gsub(/`/, '\\\\`')
.gsub(DOLLAR_SIGN_WITHOUT_SHELL_EXPANSION_REGEX, '\$')