Use env files for secrets

Add env files back in for secrets - hides them from process lists and
allows you to pick up the latest env file when running
`kamal app exec` without reusing.

lib/kamal/cli/accessory.rb

@@ -12,6 +12,8 @@ class Kamal::Cli::Accessory < Kamal::Cli::Base
           on(hosts) do
             execute *KAMAL.registry.login if login
             execute *KAMAL.auditor.record("Booted #{name} accessory"), verbosity: :debug
+            execute *accessory.ensure_env_directory
+            upload! accessory.secrets_io, accessory.secrets_path, mode: "0600"
             execute *accessory.run
           end
         end

lib/kamal/cli/app/boot.rb

@@ -1,6 +1,6 @@
 class Kamal::Cli::App::Boot
   attr_reader :host, :role, :version, :barrier, :sshkit
-  delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, to: :sshkit
+  delegate :execute, :capture_with_info, :capture_with_pretty_json, :info, :error, :upload!, to: :sshkit
   delegate :uses_cord?, :assets?, :running_traefik?, to: :role
 
   def initialize(host, role, sshkit, version, barrier)
@@ -48,7 +48,11 @@ class Kamal::Cli::App::Boot
 
       execute *app.tie_cord(role.cord_host_file) if uses_cord?
       hostname = "#{host.to_s[0...51].gsub(/\.+$/, '')}-#{SecureRandom.hex(6)}"
+
+      execute *app.ensure_env_directory
+      upload! role.secrets_io(host), role.secrets_path, mode: "0600"
       execute *app.run(hostname: hostname)
+
       Kamal::Cli::Healthcheck::Poller.wait_for_healthy(pause_after_ready: true) { capture_with_info(*app.status(version: version)) }
     end
 

lib/kamal/cli/lock.rb

@@ -3,7 +3,6 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   def status
     handle_missing_lock do
       on(KAMAL.primary_host) do
-        execute *KAMAL.server.ensure_run_directory
         puts capture_with_debug(*KAMAL.lock.status)
       end
     end
@@ -17,7 +16,6 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
 
     raise_if_locked do
       on(KAMAL.primary_host) do
-        execute *KAMAL.server.ensure_run_directory
         execute *KAMAL.lock.acquire(message, KAMAL.config.version), verbosity: :debug
       end
       say "Acquired the deploy lock"
@@ -28,7 +26,6 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   def release
     handle_missing_lock do
       on(KAMAL.primary_host) do
-        execute *KAMAL.server.ensure_run_directory
         execute *KAMAL.lock.release, verbosity: :debug
       end
       say "Released the deploy lock"

lib/kamal/cli/traefik.rb

@@ -4,6 +4,8 @@ class Kamal::Cli::Traefik < Kamal::Cli::Base
     with_lock do
       on(KAMAL.traefik_hosts) do
         execute *KAMAL.registry.login
+        execute *KAMAL.traefik.ensure_env_directory
+        upload! KAMAL.traefik.secrets_io, KAMAL.traefik.secrets_path, mode: "0600"
         execute *KAMAL.traefik.start_or_run
       end
     end

lib/kamal/commands/accessory.rb

@@ -1,7 +1,9 @@
 class Kamal::Commands::Accessory < Kamal::Commands::Base
   attr_reader :accessory_config
   delegate :service_name, :image, :hosts, :port, :files, :directories, :cmd,
-           :publish_args, :env_args, :volume_args, :label_args, :option_args, to: :accessory_config
+           :publish_args, :env_args, :volume_args, :label_args, :option_args,
+           :secrets_io, :secrets_path, :env_directory,
+           to: :accessory_config
 
   def initialize(config, name:)
     super(config)
@@ -98,6 +100,10 @@ class Kamal::Commands::Accessory < Kamal::Commands::Base
     docker :image, :rm, "--force", image
   end
 
+  def ensure_env_directory
+    make_directory env_directory
+  end
+
   private
     def service_filter
       [ "--filter", "label=service=#{service_name}" ]

lib/kamal/commands/app.rb

@@ -69,6 +69,10 @@ class Kamal::Commands::App < Kamal::Commands::Base
       extract_version_from_name
   end
 
+  def ensure_env_directory
+    make_directory role.env_directory
+  end
+
   private
     def container_name(version = nil)
       [ role.container_prefix, version || config.version ].compact.join("-")

lib/kamal/commands/base.rb

@@ -37,6 +37,10 @@ module Kamal::Commands
       [ :rm, "-r", path ]
     end
 
+    def remove_file(path)
+      [ :rm, path ]
+    end
+
     private
       def combine(*commands, by: "&&")
         commands

lib/kamal/commands/traefik.rb

@@ -1,6 +1,6 @@
 class Kamal::Commands::Traefik < Kamal::Commands::Base
   delegate :argumentize, :optionize, to: Kamal::Utils
-  delegate :port, :publish?, :labels, :env, :image, :options, :args, to: :"config.traefik"
+  delegate :port, :publish?, :labels, :env, :image, :options, :args, :env_args, :secrets_io, :env_directory, :secrets_path, to: :"config.traefik"
 
   def run
     docker :run, "--name traefik",
@@ -54,6 +54,10 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
     docker :image, :prune, "--all", "--force", "--filter", "label=org.opencontainers.image.title=Traefik"
   end
 
+  def ensure_env_directory
+    make_directory env_directory
+  end
+
   private
     def publish_args
       argumentize "--publish", port if publish?
@@ -63,10 +67,6 @@ class Kamal::Commands::Traefik < Kamal::Commands::Base
       argumentize "--label", labels
     end
 
-    def env_args
-      env.args
-    end
-
     def docker_options_args
       optionize(options)
     end

lib/kamal/configuration.rb

@@ -217,7 +217,7 @@ class Kamal::Configuration
   end
 
 
-  def host_env_directory
+  def env_directory
     File.join(run_directory, "env")
   end
 

lib/kamal/configuration/accessory.rb

@@ -51,7 +51,19 @@ class Kamal::Configuration::Accessory
   end
 
   def env_args
-    env.args
+    [ *env.clear_args, *argumentize("--env-file", secrets_path) ]
+  end
+
+  def env_directory
+    File.join(config.env_directory, "accessories")
+  end
+
+  def secrets_io
+    env.secrets_io
+  end
+
+  def secrets_path
+    File.join(config.env_directory, "accessories", "#{service_name}.env")
   end
 
   def files

lib/kamal/configuration/env.rb

@@ -13,8 +13,12 @@ class Kamal::Configuration::Env
     validate! config, context: context, with: Kamal::Configuration::Validator::Env
   end
 
-  def args
-    [ *clear_args, *secret_args ]
+  def clear_args
+    argumentize("--env", clear)
+  end
+
+  def secrets_io
+    Kamal::EnvFile.new(secret_keys.to_h { |key| [ key, secrets[key] ] }).to_io
   end
 
   def merge(other)
@@ -22,13 +26,4 @@ class Kamal::Configuration::Env
       config: { "clear" => clear.merge(other.clear), "secret" => secret_keys | other.secret_keys },
       secrets: secrets
   end
-
-  private
-    def clear_args
-      argumentize("--env", clear)
-    end
-
-    def secret_args
-      argumentize("--env", secret_keys.to_h { |key| [ key, secrets[key] ] }, sensitive: true)
-    end
 end

lib/kamal/configuration/role.rb

@@ -77,7 +77,19 @@ class Kamal::Configuration::Role
   end
 
   def env_args(host)
-    env(host).args
+    [ *env(host).clear_args, *argumentize("--env-file", secrets_path) ]
+  end
+
+  def env_directory
+    File.join(config.env_directory, "roles")
+  end
+
+  def secrets_io(host)
+    env(host).secrets_io
+  end
+
+  def secrets_path
+    File.join(config.env_directory, "roles", "#{container_prefix}.env")
   end
 
   def asset_volume_args

lib/kamal/configuration/traefik.rb

@@ -1,4 +1,6 @@
 class Kamal::Configuration::Traefik
+  delegate :argumentize, to: Kamal::Utils
+
   DEFAULT_IMAGE = "traefik:v2.10"
   CONTAINER_PORT = 80
   DEFAULT_ARGS = {
@@ -57,4 +59,20 @@ class Kamal::Configuration::Traefik
   def image
     traefik_config.fetch("image", DEFAULT_IMAGE)
   end
+
+  def env_args
+    [ *env.clear_args, *argumentize("--env-file", secrets_path) ]
+  end
+
+  def env_directory
+    File.join(config.env_directory, "traefik")
+  end
+
+  def secrets_io
+    env.secrets_io
+  end
+
+  def secrets_path
+    File.join(config.env_directory, "traefik", "traefik.env")
+  end
 end

lib/kamal/env_file.rb

@@ -0,0 +1,42 @@
+# Encode an env hash as a string where secret values have been looked up and all values escaped for Docker.
+class Kamal::EnvFile
+  def initialize(env)
+    @env = env
+  end
+
+  def to_s
+    env_file = StringIO.new.tap do |contents|
+      @env.each do |key, value|
+        contents << docker_env_file_line(key, value)
+      end
+    end.string
+
+    # Ensure the file has some contents to avoid the SSHKIT empty file warning
+    env_file.presence || "\n"
+  end
+
+  def to_io
+    StringIO.new(to_s)
+  end
+
+  alias to_str to_s
+
+  private
+    def docker_env_file_line(key, value)
+      "#{key}=#{escape_docker_env_file_value(value)}\n"
+    end
+
+    # Escape a value to make it safe to dump in a docker file.
+    def escape_docker_env_file_value(value)
+      # keep non-ascii(UTF-8) characters as it is
+      value.to_s.scan(/[\x00-\x7F]+|[^\x00-\x7F]+/).map do |part|
+        part.ascii_only? ? escape_docker_env_file_ascii_value(part) : part
+      end.join
+    end
+
+    def escape_docker_env_file_ascii_value(value)
+      # Doublequotes are treated literally in docker env files
+      # so remove leading and trailing ones and unescape any others
+      value.to_s.dump[1..-2].gsub(/\\"/, "\"")
+    end
+end