Replace .env* with .kamal/env*

By default look for the env file in .kamal/env to avoid clashes with
other tools using .env.

For now we'll still load .env and issue a deprecation warning, but in
future we'll stop reading those.

Dockerfile

@@ -14,7 +14,7 @@ COPY Gemfile Gemfile.lock kamal.gemspec ./
 COPY lib/kamal/version.rb /kamal/lib/kamal/version.rb
 
 # Install system dependencies
-RUN apk add --no-cache --update build-base git docker openrc openssh-client-default \
+RUN apk add --no-cache build-base git docker openrc openssh-client-default \
     && rc-update add docker boot \
     && gem install bundler --version=2.4.3 \
     && bundle install

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kamal (1.8.0)
+    kamal (1.8.1)
       activesupport (>= 7.0)
       base64 (~> 0.2)
       bcrypt_pbkdf (~> 1.0)

lib/kamal/cli/base.rb

@@ -37,11 +37,24 @@ module Kamal::Cli
 
       def load_env
         if destination = options[:destination]
-          Dotenv.load(".env.#{destination}", ".env")
+          if File.exist?(".kamal/env.#{destination}") || File.exist?(".kamal/env")
+            Dotenv.load(".kamal/env.#{destination}", ".kamal/env")
           else
+            loading_files = [ (".env" if File.exist?(".env")), (".env.#{destination}" if File.exist?(".env.#{destination}")) ].compact
+            if loading_files.any?
+              warn "Loading #{loading_files.join(" and ")} from the project root, use .kamal/env* instead"
+              Dotenv.load(".env.#{destination}", ".env")
+            end
+          end
+        else
+          if File.exist?(".kamal/env")
+            Dotenv.load(".kamal/env")
+          elsif File.exist?(".env")
+            warn "Loading .env from the project root is deprecated, use .kamal/env instead"
             Dotenv.load(".env")
           end
         end
+      end
 
       def reset_env
         replace_env @original_env

lib/kamal/cli/build.rb

@@ -140,7 +140,7 @@ class Kamal::Cli::Build < Kamal::Cli::Base
         mirror_hosts = Concurrent::Hash.new
         on(KAMAL.hosts) do |host|
           first_mirror = capture_with_info(*KAMAL.builder.first_mirror).strip.presence
-          mirror_hosts[first_mirror] ||= host if first_mirror
+          mirror_hosts[first_mirror] ||= host.to_s if first_mirror
         rescue SSHKit::Command::Failed => e
           raise unless e.message =~ /error calling index: reflect: slice index out of range/
         end

lib/kamal/cli/main.rb

@@ -182,6 +182,15 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)"
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip .env file push"
   def envify
+    if destination = options[:destination]
+      env_template_path = ".kamal/env.#{destination}.erb"
+      env_path          = ".kamal/env.#{destination}"
+    else
+      env_template_path = ".kamal/env.erb"
+      env_path          = ".kamal/env"
+    end
+
+    unless Pathname.new(File.expand_path(env_template_path)).exist?
       if destination = options[:destination]
         env_template_path = ".env.#{destination}.erb"
         env_path          = ".env.#{destination}"
@@ -190,6 +199,11 @@ class Kamal::Cli::Main < Kamal::Cli::Base
         env_path          = ".env"
       end
 
+      if Pathname.new(File.expand_path(env_template_path)).exist?
+        warn "Loading #{env_template_path} from the project root is deprecated, use .kamal/env[.<DESTINATION>].erb instead"
+      end
+    end
+
     if Pathname.new(File.expand_path(env_template_path)).exist?
       # Ensure existing env doesn't pollute template evaluation
       content = with_original_env { ERB.new(File.read(env_template_path), trim_mode: "-").result }

lib/kamal/configuration.rb

@@ -47,7 +47,7 @@ class Kamal::Configuration
     @destination = destination
     @declared_version = version
 
-    validate! raw_config, example: validation_yml.symbolize_keys, context: ""
+    validate! raw_config, example: validation_yml.symbolize_keys, context: "", with: Kamal::Configuration::Validator::Configuration
 
     # Eager load config to validate it, these are first as they have dependencies later on
     @servers = Servers.new(config: self)

lib/kamal/configuration/docs/configuration.yml

@@ -2,13 +2,24 @@
 #
 # Configuration is read from the `config/deploy.yml`
 #
+
+# Destinations
+#
 # When running commands, you can specify a destination with the `-d` flag,
 # e.g. `kamal deploy -d staging`
 #
 # In this case the configuration will also be read from `config/deploy.staging.yml`
 # and merged with the base configuration.
+
+# Extensions
 #
-# The available configuration options are explained below.
+# Kamal will not accept unrecognized keys in the configuration file.
+#
+# However, you might want to declare a configuration block using YAML anchors
+# and aliases to avoid repetition.
+#
+# You can use prefix a configuration section with `x-` to indicate that it is an
+# extension. Kamal will ignore the extension and not raise an error.
 
 # The service name
 # This is a required value. It is used as the container name prefix.

lib/kamal/configuration/validator.rb

@@ -15,11 +15,10 @@ class Kamal::Configuration::Validator
     def validate_against_example!(validation_config, example)
       validate_type! validation_config, Hash
 
-      if (unknown_keys = validation_config.keys - example.keys).any?
+      check_unknown_keys! validation_config, example
-        unknown_keys_error unknown_keys
-      end
 
       validation_config.each do |key, value|
+        next if extension?(key)
         with_context(key) do
           example_value = example[key]
 
@@ -137,4 +136,18 @@ class Kamal::Configuration::Validator
     ensure
       @context = old_context
     end
+
+    def allow_extensions?
+      false
+    end
+
+    def extension?(key)
+      key.to_s.start_with?("x-")
+    end
+
+    def check_unknown_keys!(config, example)
+      unknown_keys = config.keys - example.keys
+      unknown_keys.reject! { |key| extension?(key) } if allow_extensions?
+      unknown_keys_error unknown_keys if unknown_keys.present?
+    end
 end

lib/kamal/configuration/validator/configuration.rb

@@ -0,0 +1,6 @@
+class Kamal::Configuration::Validator::Configuration < Kamal::Configuration::Validator
+  private
+    def allow_extensions?
+      true
+    end
+end

lib/kamal/version.rb

@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "1.8.0"
+  VERSION = "1.8.1"
 end

test/cli/build_test.rb

@@ -185,7 +185,7 @@ class CliBuildTest < CliTestCase
     run_command("pull").tap do |output|
       assert_match /Pulling image on 1\.1\.1\.\d to seed the mirror\.\.\./, output
       assert_match "Pulling image on remaining hosts...", output
-      assert_match /docker pull dhh\/app:999/, output
+      assert_equal 4, output.scan(/docker pull dhh\/app:999/).size, output
       assert_match "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:999 | grep -x app || (echo \"Image dhh/app:999 is missing the 'service' label\" && exit 1)", output
     end
   end
@@ -199,7 +199,7 @@ class CliBuildTest < CliTestCase
     run_command("pull").tap do |output|
       assert_match /Pulling image on 1\.1\.1\.\d, 1\.1\.1\.\d to seed the mirrors\.\.\./, output
       assert_match "Pulling image on remaining hosts...", output
-      assert_match /docker pull dhh\/app:999/, output
+      assert_equal 4, output.scan(/docker pull dhh\/app:999/).size, output
       assert_match "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:999 | grep -x app || (echo \"Image dhh/app:999 is missing the 'service' label\" && exit 1)", output
     end
   end

test/cli/cli_test_case.rb

@@ -42,12 +42,13 @@ class CliTestCase < ActiveSupport::TestCase
     end
 
     def assert_hook_ran(hook, output, version:, service_version:, hosts:, command:, subcommand: nil, runtime: false)
-      performer = Kamal::Git.email.presence || `whoami`.chomp
+      whoami = `whoami`.chomp
+      performer = Kamal::Git.email.presence || whoami
       service = service_version.split("@").first
 
       assert_match "Running the #{hook} hook...\n", output
 
-      expected = %r{Running\s/usr/bin/env\s\.kamal/hooks/#{hook}\sas\s#{performer}@localhost\n\s
+      expected = %r{Running\s/usr/bin/env\s\.kamal/hooks/#{hook}\sas\s#{whoami}@localhost\n\s
         DEBUG\s\[[0-9a-f]*\]\sCommand:\s\(\sexport\s
         KAMAL_RECORDED_AT=\"\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ\"\s
         KAMAL_PERFORMER=\"#{performer}\"\s

test/cli/main_test.rb

@@ -447,9 +447,9 @@ class CliMainTest < CliTestCase
   end
 
   test "envify" do
-    with_test_dotenv(".env.erb": "HELLO=<%= 'world' %>") do
+    with_test_env_files("env.erb": "HELLO=<%= 'world' %>") do
       run_command("envify")
-      assert_equal("HELLO=world", File.read(".env"))
+      assert_equal("HELLO=world", File.read(".kamal/env"))
     end
   end
 
@@ -461,32 +461,32 @@ class CliMainTest < CliTestCase
       <% end  -%>
     EOF
 
-    with_test_dotenv(".env.erb": file) do
+    with_test_env_files("env.erb": file) do
       run_command("envify")
-      assert_equal("HELLO=world\nKEY=value\n", File.read(".env"))
+      assert_equal("HELLO=world\nKEY=value\n", File.read(".kamal/env"))
     end
   end
 
   test "envify with destination" do
-    with_test_dotenv(".env.world.erb": "HELLO=<%= 'world' %>") do
+    with_test_env_files("env.world.erb": "HELLO=<%= 'world' %>") do
       run_command("envify", "-d", "world", config_file: "deploy_for_dest")
-      assert_equal "HELLO=world", File.read(".env.world")
+      assert_equal "HELLO=world", File.read(".kamal/env.world")
     end
   end
 
   test "envify with skip_push" do
-    Pathname.any_instance.expects(:exist?).returns(true).times(1)
+    Pathname.any_instance.expects(:exist?).returns(true).times(2)
-    File.expects(:read).with(".env.erb").returns("HELLO=<%= 'world' %>")
+    File.expects(:read).with(".kamal/env.erb").returns("HELLO=<%= 'world' %>")
-    File.expects(:write).with(".env", "HELLO=world", perm: 0600)
+    File.expects(:write).with(".kamal/env", "HELLO=world", perm: 0600)
 
     Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push").never
     run_command("envify", "--skip-push")
   end
 
   test "envify with clean env" do
-    with_test_dotenv(".env": "HELLO=already", ".env.erb": "HELLO=<%= ENV.fetch 'HELLO', 'never' %>") do
+    with_test_env_files("env": "HELLO=already", "env.erb": "HELLO=<%= ENV.fetch 'HELLO', 'never' %>") do
       run_command("envify", "--skip-push")
-      assert_equal "HELLO=never", File.read(".env")
+      assert_equal "HELLO=never", File.read(".kamal/env")
     end
   end
 
@@ -542,16 +542,19 @@ class CliMainTest < CliTestCase
       stdouted { Kamal::Cli::Main.start([ *command, "-c", "test/fixtures/#{config_file}.yml" ]) }
     end
 
-    def with_test_dotenv(**files)
+    def with_test_env_files(**files)
       Dir.mktmpdir do |dir|
         fixtures_dup = File.join(dir, "test")
         FileUtils.mkdir_p(fixtures_dup)
         FileUtils.cp_r("test/fixtures/", fixtures_dup)
 
         Dir.chdir(dir) do
+          FileUtils.mkdir_p(".kamal")
+          Dir.chdir(".kamal") do
             files.each do |filename, contents|
               File.binwrite(filename.to_s, contents)
             end
+          end
           yield
         end
       end

test/configuration_test.rb

@@ -344,4 +344,12 @@ class ConfigurationTest < ActiveSupport::TestCase
 
     assert_raises(Kamal::ConfigurationError) { Kamal::Configuration.new(@deploy_with_roles.merge(retain_containers: 0)) }
   end
+
+  test "extensions" do
+    dest_config_file = Pathname.new(File.expand_path("fixtures/deploy_with_extensions.yml", __dir__))
+
+    config = Kamal::Configuration.create_from config_file: dest_config_file
+    assert_equal config.role(:web_tokyo).running_traefik?, true
+    assert_equal config.role(:web_chicago).running_traefik?, true
+  end
 end

test/fixtures/deploy_with_extensions.yml

@@ -0,0 +1,24 @@
+
+x-web: &web
+  traefik: true
+
+service: app
+image: dhh/app
+servers:
+  web_chicago:
+    <<: *web
+    hosts:
+      - 1.1.1.1
+      - 1.1.1.2
+  web_tokyo:
+    <<: *web
+    hosts:
+      - 1.1.1.3
+      - 1.1.1.4
+env:
+  REDIS_URL: redis://x/y
+registry:
+  server: registry.digitalocean.com
+  username: user
+  password: pw
+primary_role: web_tokyo

test/integration/main_test.rb

@@ -97,7 +97,7 @@ class MainTest < IntegrationTest
 
   private
     def assert_local_env_file(contents)
-      assert_equal contents, deployer_exec("cat .env", capture: true)
+      assert_equal contents, deployer_exec("cat .kamal/env", capture: true)
     end
 
     def assert_envs(version:)
@@ -127,7 +127,7 @@ class MainTest < IntegrationTest
     end
 
     def remove_local_env_file
-      deployer_exec("rm .env")
+      deployer_exec("rm .kamal/env")
     end
 
     def assert_remote_env_file(contents, vm:)