Lazily load secrets whenever needed

lib/kamal.rb

@@ -5,6 +5,7 @@ end
 require "active_support"
 require "zeitwerk"
 require "yaml"
+require "tmpdir"
 
 loader = Zeitwerk::Loader.for_gem
 loader.ignore(File.join(__dir__, "kamal", "sshkit_with_ext.rb"))

lib/kamal/cli/app/boot.rb

@@ -88,8 +88,12 @@ class Kamal::Cli::App::Boot
     def close_barrier
       if barrier.close
         info "First #{KAMAL.primary_role} container is unhealthy on #{host}, not booting any other roles"
-        error capture_with_info(*app.logs(version: version))
-        error capture_with_info(*app.container_health_log(version: version))
+        begin
+          error capture_with_info(*app.logs(version: version))
+          error capture_with_info(*app.container_health_log(version: version))
+        rescue SSHKit::Command::Failed
+          error "Could not fetch logs for #{version}"
+        end
       end
     end
 

lib/kamal/cli/base.rb

@@ -31,24 +31,15 @@ module Kamal::Cli
       else
         super
       end
-      @original_env = ENV.to_h.dup
-      initialize_commander(options_with_subcommand_class_options)
+      initialize_commander unless KAMAL.configured?
     end
 
     private
-      def load_secrets
-        if destination = options[:destination]
-          Dotenv.parse(".kamal/secrets.#{destination}", ".kamal/secrets")
-        else
-          Dotenv.parse(".kamal/secrets")
-        end
-      end
-
       def options_with_subcommand_class_options
         options.merge(@_initializer.last[:class_options] || {})
       end
 
-      def initialize_commander(options)
+      def initialize_commander
         KAMAL.tap do |commander|
           if options[:verbose]
             ENV["VERBOSE"] = "1" # For backtraces via cli/start

lib/kamal/cli/build.rb

@@ -51,7 +51,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base
       push = KAMAL.builder.push
 
       KAMAL.with_verbosity(:debug) do
-        Dir.chdir(KAMAL.config.builder.build_directory) { execute *push }
+        Dir.chdir(KAMAL.config.builder.build_directory) { execute *push, env: KAMAL.config.builder.secrets }
       end
     end
   end

lib/kamal/cli/main.rb

@@ -202,9 +202,6 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   desc "build", "Build application image"
   subcommand "build", Kamal::Cli::Build
 
-  desc "env", "Manage environment files"
-  subcommand "env", Kamal::Cli::Env
-
   desc "lock", "Manage the deploy lock"
   subcommand "lock", Kamal::Cli::Lock
 

lib/kamal/commander.rb

@@ -23,6 +23,10 @@ class Kamal::Commander
     @config, @config_kwargs = nil, kwargs
   end
 
+  def configured?
+    @config || @config_kwargs
+  end
+
   attr_reader :specific_roles, :specific_hosts
 
   def specific_primary!

lib/kamal/commands/accessory.rb

@@ -98,14 +98,6 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     docker :image, :rm, "--force", image
   end
 
-  def make_env_directory
-    make_directory accessory_config.env.secrets_directory
-  end
-
-  def remove_env_file
-    [ :rm, "-f", accessory_config.env.secrets_file ]
-  end
-
   private
     def service_filter
       [ "--filter", "label=service=#{service_name}" ]

lib/kamal/commands/app.rb

@@ -69,16 +69,6 @@ class Kamal::Commands::App < Kamal::Commands::Base
       extract_version_from_name
   end
 
-
-  def make_env_directory
-    make_directory role.env(host).secrets_directory
-  end
-
-  def remove_env_file
-    [ :rm, "-f", role.env(host).secrets_file ]
-  end
-
-
   private
     def container_name(version = nil)
       [ role.container_prefix, version || config.version ].compact.join("-")

lib/kamal/commands/auditor.rb

@@ -8,9 +8,12 @@ class Kamal::Commands::Auditor < Kamal::Commands::Base
 
   # Runs remotely
   def record(line, **details)
-    append \
-      [ :echo, audit_tags(**details).except(:version, :service_version, :service).to_s, line ],
-      audit_log_file
+    combine \
+      [ :mkdir, "-p", config.run_directory ],
+      append(
+        [ :echo, audit_tags(**details).except(:version, :service_version, :service).to_s, line ],
+        audit_log_file
+      )
   end
 
   def reveal

lib/kamal/commands/builder/base.rb

@@ -78,7 +78,7 @@ class Kamal::Commands::Builder::Base < Kamal::Commands::Base
     end
 
     def build_secrets
-      argumentize "--secret", secrets.collect { |secret| [ "id", secret ] }
+      argumentize "--secret", secrets.keys.collect { |secret| [ "id", secret ] }
     end
 
     def build_dockerfile

lib/kamal/commands/traefik.rb

@@ -54,14 +54,6 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
     docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
   end
 
-  def make_env_directory
-    make_directory(env.secrets_directory)
-  end
-
-  def remove_env_file
-    [ :rm, "-f", env.secrets_file ]
-  end
-
   private
     def publish_args
       argumentize "--publish", port if publish?

lib/kamal/configuration.rb

@@ -57,7 +57,7 @@ class Kamal::Configuration
     @aliases = @raw_config.aliases&.keys&.to_h { |name| [ name, Alias.new(name, config: self) ] } || {}
     @boot = Boot.new(config: self)
     @builder = Builder.new(config: self)
-    @env = Env.new(config: @raw_config.env || {})
+    @env = Env.new(config: @raw_config.env || {}, secrets: secrets)
 
     @healthcheck = Healthcheck.new(healthcheck_config: @raw_config.healthcheck)
     @logging = Logging.new(logging_config: @raw_config.logging)
@@ -224,7 +224,7 @@ class Kamal::Configuration
 
   def env_tags
     @env_tags ||= if (tags = raw_config.env["tags"])
-      tags.collect { |name, config| Env::Tag.new(name, config: config) }
+      tags.collect { |name, config| Env::Tag.new(name, config: config, secrets: secrets) }
     else
       []
     end
@@ -254,6 +254,10 @@ class Kamal::Configuration
     }.compact
   end
 
+  def secrets
+    @secrets ||= Secrets.new(destination: destination)
+  end
+
   private
     # Will raise ArgumentError if any required config keys are missing
     def ensure_destination_if_required

lib/kamal/configuration/accessory.rb

@@ -16,7 +16,7 @@ class Kamal::Configuration::Accessory
 
     @env = Kamal::Configuration::Env.new \
       config: accessory_config.fetch("env", {}),
-      secrets_file: File.join(config.host_env_directory, "accessories", "#{service_name}.env"),
+      secrets: config.secrets,
       context: "accessories/#{name}/env"
   end
 

lib/kamal/configuration/builder.rb

@@ -62,7 +62,7 @@ class Kamal::Configuration::Builder
   end
 
   def secrets
-    builder_config["secrets"] || []
+    (builder_config["secrets"] || []).to_h { |key| [ key, config.secrets[key] ] }
   end
 
   def dockerfile

lib/kamal/configuration/env.rb

@@ -1,36 +1,34 @@
 class Kamal::Configuration::Env
   include Kamal::Configuration::Validation
 
-  attr_reader :secrets_keys, :clear, :secrets_file, :context
+  attr_reader :context, :secrets
+  attr_reader :clear, :secret_keys
   delegate :argumentize, to: Kamal::Utils
 
-  def initialize(config:, secrets_file: nil, context: "env")
+  def initialize(config:, secrets:, context: "env")
     @clear = config.fetch("clear", config.key?("secret") || config.key?("tags") ? {} : config)
-    @secrets_keys = config.fetch("secret", [])
-    @secrets_file = secrets_file
+    @secrets = secrets
+    @secret_keys = config.fetch("secret", [])
     @context = context
     validate! config, context: context, with: Kamal::Configuration::Validator::Env
   end
 
   def args
-    [ "--env-file", secrets_file, *argumentize("--env", clear) ]
-  end
-
-  def secrets_io
-    StringIO.new(Kamal::EnvFile.new(secrets).to_s)
-  end
-
-  def secrets
-    @secrets ||= secrets_keys.to_h { |key| [ key, ENV.fetch(key) ] }
-  end
-
-  def secrets_directory
-    File.dirname(secrets_file)
+    [ *clear_args, *secret_args ]
   end
 
   def merge(other)
     self.class.new \
-      config: { "clear" => clear.merge(other.clear), "secret" => secrets_keys | other.secrets_keys },
-      secrets_file: secrets_file || other.secrets_file
+      config: { "clear" => clear.merge(other.clear), "secret" => secret_keys | other.secret_keys },
+      secrets: secrets
   end
+
+  private
+    def clear_args
+      argumentize("--env", clear)
+    end
+
+    def secret_args
+      argumentize("--env", secret_keys.to_h { |key| [ key, secrets[key] ] }, sensitive: true)
+    end
 end

lib/kamal/configuration/env/tag.rb

@@ -1,12 +1,13 @@
 class Kamal::Configuration::Env::Tag
-  attr_reader :name, :config
+  attr_reader :name, :config, :secrets
 
-  def initialize(name, config:)
+  def initialize(name, config:, secrets:)
     @name = name
     @config = config
+    @secrets = secrets
   end
 
   def env
-    Kamal::Configuration::Env.new(config: config)
+    Kamal::Configuration::Env.new(config: config, secrets: secrets)
   end
 end

lib/kamal/configuration/registry.rb

@@ -1,10 +1,11 @@
 class Kamal::Configuration::Registry
   include Kamal::Configuration::Validation
 
-  attr_reader :registry_config
+  attr_reader :registry_config, :secrets
 
   def initialize(config:)
     @registry_config = config.raw_config.registry || {}
+    @secrets = config.secrets
     validate! registry_config, with: Kamal::Configuration::Validator::Registry
   end
 
@@ -23,7 +24,7 @@ class Kamal::Configuration::Registry
   private
     def lookup(key)
       if registry_config[key].is_a?(Array)
-        ENV.fetch(registry_config[key].first).dup
+        secrets[registry_config[key].first]
       else
         registry_config[key]
       end

lib/kamal/configuration/role.rb

@@ -18,7 +18,7 @@ class Kamal::Configuration::Role
 
     @specialized_env = Kamal::Configuration::Env.new \
       config: specializations.fetch("env", {}),
-      secrets_file: File.join(config.host_env_directory, "roles", "#{container_prefix}.env"),
+      secrets: config.secrets,
       context: "servers/#{name}/env"
 
     @specialized_logging = Kamal::Configuration::Logging.new \

lib/kamal/configuration/secrets.rb

@@ -0,0 +1,25 @@
+class Kamal::Configuration::Secrets
+  attr_reader :secret_files
+
+  def initialize(destination: nil)
+    @secret_files = \
+      (destination ? [ ".kamal/secrets", ".kamal/secrets.#{destination}" ] : [ ".kamal/secrets" ])
+        .select { |file| File.exist?(file) }
+  end
+
+  def [](key)
+    @secrets ||= load
+    @secrets.fetch(key)
+  rescue KeyError
+    if secret_files.any?
+      raise Kamal::ConfigurationError, "Secret '#{key}' not found in #{secret_files.join(', ')}"
+    else
+      raise Kamal::ConfigurationError, "Secret '#{key}' not found, no secret files provided"
+    end
+  end
+
+  private
+    def load
+      secret_files.any? ? Dotenv.parse(*secret_files) : {}
+    end
+end

lib/kamal/configuration/traefik.rb

@@ -34,7 +34,7 @@ class Kamal::Configuration::Traefik
   def env
     Kamal::Configuration::Env.new \
       config: traefik_config.fetch("env", {}),
-      secrets_file: File.join(config.host_env_directory, "traefik", "traefik.env"),
+      secrets: config.secrets,
       context: "traefik/env"
   end
 

lib/kamal/env_file.rb

@@ -1,38 +0,0 @@
-# Encode an env hash as a string where secret values have been looked up and all values escaped for Docker.
-class Kamal::EnvFile
-  def initialize(env)
-    @env = env
-  end
-
-  def to_s
-    env_file = StringIO.new.tap do |contents|
-      @env.each do |key, value|
-        contents << docker_env_file_line(key, value)
-      end
-    end.string
-
-    # Ensure the file has some contents to avoid the SSHKIT empty file warning
-    env_file.presence || "\n"
-  end
-
-  alias to_str to_s
-
-  private
-    def docker_env_file_line(key, value)
-      "#{key}=#{escape_docker_env_file_value(value)}\n"
-    end
-
-    # Escape a value to make it safe to dump in a docker file.
-    def escape_docker_env_file_value(value)
-      # keep non-ascii(UTF-8) characters as it is
-      value.to_s.scan(/[\x00-\x7F]+|[^\x00-\x7F]+/).map do |part|
-        part.ascii_only? ? escape_docker_env_file_ascii_value(part) : part
-      end.join
-    end
-
-    def escape_docker_env_file_ascii_value(value)
-      # Doublequotes are treated literally in docker env files
-      # so remove leading and trailing ones and unescape any others
-      value.to_s.dump[1..-2].gsub(/\\"/, "\"")
-    end
-end

lib/kamal/utils.rb

@@ -54,6 +54,12 @@ module Kamal::Utils
 
   # Escape a value to make it safe for shell use.
   def escape_shell_value(value)
+    value.to_s.scan(/[\x00-\x7F]+|[^\x00-\x7F]+/) \
+      .map { |part| part.ascii_only? ? escape_ascii_shell_value(part) : part }
+      .join
+  end
+
+  def escape_ascii_shell_value(value)
     value.to_s.dump
       .gsub(/`/, '\\\\`')
       .gsub(DOLLAR_SIGN_WITHOUT_SHELL_EXPANSION_REGEX, '\$')