Host specific env with tags

Allow hosts to be tagged so we can have host specific env variables.

We might want host specific env variables for things like datacenter
specific tags or testing GC settings on a specific host.

Right now you either need to set up a separate role, or have the app
be host aware.

Now you can define tag env variables and assign those to hosts.

For example:
```
servers:
  - 1.1.1.1
  - 1.1.1.2: tag1
  - 1.1.1.2: tag2
  - 1.1.1.3: [ tag1, tag2 ]
env_tags:
  tag1:
    ENV1: value1
  tag2:
    ENV2: value2
```

The tag env supports the full env format, allowing you to set secret and
clear values.
This commit is contained in:
Donal McBreen
2024-05-02 10:36:15 +01:00
parent 3c8428504d
commit 6d062ce271
21 changed files with 334 additions and 69 deletions

View File

@@ -92,6 +92,32 @@ class CliAppTest < CliTestCase
end
end
test "boot with host tags" do
Object.any_instance.stubs(:sleep)
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", raise_on_non_zero_exit: false)
.returns("12345678") # running version
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-latest$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'")
.returns("running") # health check
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false)
.returns("123") # old version
SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info)
.with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-123", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", raise_on_non_zero_exit: false)
.returns("") # old version
run_command("boot", config: :with_env_tags).tap do |output|
assert_match "docker tag dhh/app:latest dhh/app:latest", output
assert_match %r{docker run --detach --restart unless-stopped --name app-web-latest --hostname 1.1.1.1-[0-9a-f]{12} -e KAMAL_CONTAINER_NAME="app-web-latest" -e KAMAL_VERSION="latest" --env-file .kamal/env/roles/app-web.env --env TEST="root" --env EXPERIMENT="disabled" --env SITE="site1"}, output
assert_match "docker container ls --all --filter name=^app-web-123$ --quiet | xargs docker stop", output
end
end
test "start" do
run_command("start").tap do |output|
assert_match "docker start app-web-999", output

View File

@@ -60,7 +60,7 @@ class CommandsAppTest < ActiveSupport::TestCase
@config[:servers] = { "web" => [ "1.1.1.1" ], "jobs" => { "hosts" => [ "1.1.1.2" ], "cmd" => "bin/jobs", "options" => { "mount" => "somewhere", "cap-add" => true } } }
assert_equal \
"docker run --detach --restart unless-stopped --name app-jobs-999 -e KAMAL_CONTAINER_NAME=\"app-jobs-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-jobs.env --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"jobs\" --label destination --mount \"somewhere\" --cap-add dhh/app:999 bin/jobs",
new_command(role: "jobs").run.join(" ")
new_command(role: "jobs", host: "1.1.1.2").run.join(" ")
end
test "run with logging config" do
@@ -80,6 +80,15 @@ class CommandsAppTest < ActiveSupport::TestCase
new_command.run.join(" ")
end
test "run with tags" do
@config[:servers] = [ { "1.1.1.1" => "tag1" } ]
@config[:env_tags] = { "tag1" => { "ENV1" => "value1" } }
assert_equal \
"docker run --detach --restart unless-stopped --name app-web-999 -e KAMAL_CONTAINER_NAME=\"app-web-999\" -e KAMAL_VERSION=\"999\" --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" --health-cmd \"(curl -f http://localhost:3000/up || exit 1) && (stat /tmp/kamal-cord/cord > /dev/null || exit 1)\" --health-interval \"1s\" --volume $(pwd)/.kamal/cords/app-web-12345678901234567890123456789012:/tmp/kamal-cord --log-opt max-size=\"10m\" --label service=\"app\" --label role=\"web\" --label destination --label traefik.http.services.app-web.loadbalancer.server.scheme=\"http\" --label traefik.http.routers.app-web.rule=\"PathPrefix(\\`/\\`)\" --label traefik.http.routers.app-web.priority=\"2\" --label traefik.http.middlewares.app-web-retry.retry.attempts=\"5\" --label traefik.http.middlewares.app-web-retry.retry.initialinterval=\"500ms\" --label traefik.http.routers.app-web.middlewares=\"app-web-retry@docker\" dhh/app:999",
new_command.run.join(" ")
end
test "start" do
assert_equal \
"docker start app-web-999",
@@ -183,6 +192,15 @@ class CommandsAppTest < ActiveSupport::TestCase
new_command.execute_in_new_container("bin/rails", "db:setup", env: { "foo" => "bar" }).join(" ")
end
test "execute in new container with tags" do
@config[:servers] = [ { "1.1.1.1" => "tag1" } ]
@config[:env_tags] = { "tag1" => { "ENV1" => "value1" } }
assert_equal \
"docker run --rm --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" dhh/app:999 bin/rails db:setup",
new_command.execute_in_new_container("bin/rails", "db:setup", env: {}).join(" ")
end
test "execute in new container with custom options" do
@config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } }
assert_equal \
@@ -204,18 +222,26 @@ class CommandsAppTest < ActiveSupport::TestCase
test "execute in new container over ssh" do
assert_match %r{docker run -it --rm --env-file .kamal/env/roles/app-web.env dhh/app:999 bin/rails c},
new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1", env: {})
new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {})
end
test "execute in new container over ssh with tags" do
@config[:servers] = [ { "1.1.1.1" => "tag1" } ]
@config[:env_tags] = { "tag1" => { "ENV1" => "value1" } }
assert_equal "ssh -t root@1.1.1.1 -p 22 'docker run -it --rm --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" dhh/app:999 bin/rails c'",
new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {})
end
test "execute in new container with custom options over ssh" do
@config[:servers] = { "web" => { "hosts" => [ "1.1.1.1" ], "options" => { "mount" => "somewhere", "cap-add" => true } } }
assert_match %r{docker run -it --rm --env-file .kamal/env/roles/app-web.env --mount \"somewhere\" --cap-add dhh/app:999 bin/rails c},
new_command.execute_in_new_container_over_ssh("bin/rails", "c", host: "app-1", env: {})
new_command.execute_in_new_container_over_ssh("bin/rails", "c", env: {})
end
test "execute in existing container over ssh" do
assert_match %r{docker exec -it app-web-999 bin/rails c},
new_command.execute_in_existing_container_over_ssh("bin/rails", "c", host: "app-1", env: {})
new_command.execute_in_existing_container_over_ssh("bin/rails", "c", env: {})
end
test "run over ssh" do
@@ -418,8 +444,8 @@ class CommandsAppTest < ActiveSupport::TestCase
end
private
def new_command(role: "web", **additional_config)
def new_command(role: "web", host: "1.1.1.1", **additional_config)
config = Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999")
Kamal::Commands::App.new(config, role: config.role(role))
Kamal::Commands::App.new(config, role: config.role(role), host: host)
end
end

View File

@@ -46,6 +46,15 @@ class CommandsHealthcheckTest < ActiveSupport::TestCase
new_command.run.join(" ")
end
test "run with tags" do
@config[:servers] = [ { "1.1.1.1" => "tag1" } ]
@config[:env_tags] = { "tag1" => { "ENV1" => "value1" } }
assert_equal \
"docker run --detach --name healthcheck-app-123 --publish 3999:3000 --label service=healthcheck-app -e KAMAL_CONTAINER_NAME=\"healthcheck-app\" --env-file .kamal/env/roles/app-web.env --env ENV1=\"value1\" --health-cmd \"curl -f http://localhost:3000/up || exit 1\" --health-interval \"1s\" dhh/app:123",
new_command.run.join(" ")
end
test "status" do
assert_equal \
"docker container ls --all --filter name=^healthcheck-app-123$ --quiet | xargs docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'",

102
test/configuration/env/tags_test.rb vendored Normal file
View File

@@ -0,0 +1,102 @@
require "test_helper"
class ConfigurationEnvTagsTest < ActiveSupport::TestCase
setup do
@deploy = {
service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
servers: [ { "1.1.1.1" => "odd" }, { "1.1.1.2" => "even" }, { "1.1.1.3" => [ "odd", "three" ] } ],
env: { "REDIS_URL" => "redis://x/y", "THREE" => "false" },
env_tags: {
"odd" => { "TYPE" => "odd" },
"even" => { "TYPE" => "even" },
"three" => { "THREE" => "true" }
}
}
@config = Kamal::Configuration.new(@deploy)
@deploy_with_roles = @deploy.dup.merge({
servers: {
"web" => [ { "1.1.1.1" => "odd" }, "1.1.1.2" ],
"workers" => {
"hosts" => [ { "1.1.1.3" => [ "odd", "oddjob" ] }, "1.1.1.4" ],
"cmd" => "bin/jobs",
"env" => {
"REDIS_URL" => "redis://a/b",
"WEB_CONCURRENCY" => 4
}
}
},
env_tags: {
"odd" => { "TYPE" => "odd" },
"oddjob" => { "TYPE" => "oddjob" }
}
})
@config_with_roles = Kamal::Configuration.new(@deploy_with_roles)
end
test "tags" do
assert_equal 3, @config.env_tags.size
assert_equal %w[ odd even three ], @config.env_tags.map(&:name)
assert_equal({ "TYPE" => "odd" }, @config.env_tag("odd").env.clear)
assert_equal({ "TYPE" => "even" }, @config.env_tag("even").env.clear)
assert_equal({ "THREE" => "true" }, @config.env_tag("three").env.clear)
end
test "tags with roles" do
assert_equal 2, @config_with_roles.env_tags.size
assert_equal %w[ odd oddjob ], @config_with_roles.env_tags.map(&:name)
assert_equal({ "TYPE" => "odd" }, @config_with_roles.env_tag("odd").env.clear)
assert_equal({ "TYPE" => "oddjob" }, @config_with_roles.env_tag("oddjob").env.clear)
end
test "tag overrides env" do
assert_equal "false", @config.role("web").env("1.1.1.1").clear["THREE"]
assert_equal "true", @config.role("web").env("1.1.1.3").clear["THREE"]
end
test "later tag wins" do
deploy = {
service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
servers: [ { "1.1.1.1" => [ "first", "second" ] } ],
env_tags: {
"first" => { "TYPE" => "first" },
"second" => { "TYPE" => "second" }
}
}
config = Kamal::Configuration.new(deploy)
assert_equal "second", config.role("web").env("1.1.1.1").clear["TYPE"]
end
test "tag secret env" do
ENV["PASSWORD"] = "hello"
deploy = {
service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
servers: [ { "1.1.1.1" => "secrets" } ],
env_tags: {
"secrets" => { "secret" => [ "PASSWORD" ] }
}
}
config = Kamal::Configuration.new(deploy)
assert_equal "hello", config.role("web").env("1.1.1.1").secrets["PASSWORD"]
ensure
ENV.delete "PASSWORD"
end
test "tag clear env" do
deploy = {
service: "app", image: "dhh/app", registry: { "username" => "dhh", "password" => "secret" },
servers: [ { "1.1.1.1" => "clearly" } ],
env_tags: {
"clearly" => { "clear" => { "FOO" => "bar" } }
}
}
config = Kamal::Configuration.new(deploy)
assert_equal "bar", config.role("web").env("1.1.1.1").clear["FOO"]
end
end

View File

@@ -70,10 +70,10 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
end
test "env overwritten by role" do
assert_equal "redis://a/b", @config_with_roles.role(:workers).env.clear["REDIS_URL"]
assert_equal "redis://a/b", @config_with_roles.role(:workers).env("1.1.1.3").clear["REDIS_URL"]
assert_equal "\n", @config_with_roles.role(:workers).env.secrets_io.string
assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
assert_equal "\n", @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
end
test "container name" do
@@ -86,7 +86,7 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
end
test "env args" do
assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
end
test "env secret overwritten by role" do
@@ -117,8 +117,8 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
DB_PASSWORD=secret&\"123
ENV
assert_equal expected_secrets_file, @config_with_roles.role(:workers).env.secrets_io.string
assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
ensure
ENV["REDIS_PASSWORD"] = nil
ENV["DB_PASSWORD"] = nil
@@ -141,8 +141,8 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
DB_PASSWORD=secret123
ENV
assert_equal expected_secrets_file, @config_with_roles.role(:workers).env.secrets_io.string
assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
ensure
ENV["DB_PASSWORD"] = nil
end
@@ -163,8 +163,8 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
REDIS_PASSWORD=secret456
ENV
assert_equal expected_secrets_file, @config_with_roles.role(:workers).env.secrets_io.string
assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args
assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://a/b\"", "--env", "WEB_CONCURRENCY=\"4\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
ensure
ENV["REDIS_PASSWORD"] = nil
end
@@ -191,14 +191,14 @@ class ConfigurationRoleTest < ActiveSupport::TestCase
REDIS_PASSWORD=secret456
ENV
assert_equal expected_secrets_file, @config_with_roles.role(:workers).env.secrets_io.string
assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://c/d\"" ], @config_with_roles.role(:workers).env_args
assert_equal expected_secrets_file, @config_with_roles.role(:workers).env("1.1.1.3").secrets_io.string
assert_equal [ "--env-file", ".kamal/env/roles/app-workers.env", "--env", "REDIS_URL=\"redis://c/d\"" ], @config_with_roles.role(:workers).env_args("1.1.1.3")
ensure
ENV["REDIS_PASSWORD"] = nil
end
test "env secrets_file" do
assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).env.secrets_file
assert_equal ".kamal/env/roles/app-workers.env", @config_with_roles.role(:workers).env("1.1.1.3").secrets_file
end
test "uses cord" do

29
test/fixtures/deploy_with_env_tags.yml vendored Normal file
View File

@@ -0,0 +1,29 @@
service: app
image: dhh/app
servers:
web:
- 1.1.1.1: site1
- 1.1.1.2: [ site1 experimental ]
- 1.2.1.1: site2
- 1.2.1.2: site2
workers:
- 1.1.1.3: site1
- 1.1.1.4: site1
- 1.2.1.3: site2
- 1.2.1.4: [ site2 experimental ]
env:
clear:
TEST: "root"
EXPERIMENT: "disabled"
env_tags:
site1:
SITE: site1
site2:
SITE: site2
experimental:
env:
EXPERIMENT: "enabled"
registry:
username: user
password: pw

View File

@@ -1 +1,2 @@
SECRET_TOKEN='1234 with "中文"'
SECRET_TAG='TAGME'

View File

@@ -2,13 +2,20 @@ service: app
image: app
servers:
- vm1
- vm2
- vm2: [ tag1, tag2 ]
env:
clear:
CLEAR_TOKEN: 4321
CLEAR_TAG: ""
HOST_TOKEN: "${HOST_TOKEN}"
secret:
- SECRET_TOKEN
env_tags:
tag1:
CLEAR_TAG: tagged
tag2:
secret:
- SECRET_TAG
asset_path: /usr/share/nginx/html/versions
registry:

View File

@@ -3,8 +3,7 @@ require_relative "integration_test"
class MainTest < IntegrationTest
test "envify, deploy, redeploy, rollback, details and audit" do
kamal :envify
assert_local_env_file "SECRET_TOKEN='1234 with \"中文\"'"
assert_remote_env_file "SECRET_TOKEN=1234 with \"中文\""
assert_env_files
remove_local_env_file
first_version = latest_app_version
@@ -14,9 +13,7 @@ class MainTest < IntegrationTest
kamal :deploy
assert_app_is_up version: first_version
assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy"
assert_env :CLEAR_TOKEN, "4321", version: first_version
assert_env :HOST_TOKEN, "abcd", version: first_version
assert_env :SECRET_TOKEN, "1234 with \"中文\"", version: first_version
assert_envs version: first_version
second_version = update_app_rev
@@ -97,16 +94,38 @@ class MainTest < IntegrationTest
assert_equal contents, deployer_exec("cat .env", capture: true)
end
def assert_env(key, value, version:)
assert_equal "#{key}=#{value}", docker_compose("exec vm1 docker exec app-web-#{version} env | grep #{key}", capture: true)
def assert_envs(version:)
assert_env :CLEAR_TOKEN, "4321", version: version, vm: :vm1
assert_env :HOST_TOKEN, "abcd", version: version, vm: :vm1
assert_env :SECRET_TOKEN, "1234 with \"中文\"", version: version, vm: :vm1
assert_no_env :CLEAR_TAG, version: version, vm: :vm1
assert_no_env :SECRET_TAG, version: version, vm: :vm11
assert_env :CLEAR_TAG, "tagged", version: version, vm: :vm2
assert_env :SECRET_TAG, "TAGME", version: version, vm: :vm2
end
def assert_env(key, value, vm:, version:)
assert_equal "#{key}=#{value}", docker_compose("exec #{vm} docker exec app-web-#{version} env | grep #{key}", capture: true)
end
def assert_no_env(key, vm:, version:)
assert_raises(RuntimeError, /exit 1/) do
docker_compose("exec #{vm} docker exec app-web-#{version} env | grep #{key}", capture: true)
end
end
def assert_env_files
assert_local_env_file "SECRET_TOKEN='1234 with \"中文\"'\nSECRET_TAG='TAGME'"
assert_remote_env_file "SECRET_TOKEN=1234 with \"中文\"", vm: :vm1
assert_remote_env_file "SECRET_TOKEN=1234 with \"中文\"\nSECRET_TAG=TAGME", vm: :vm2
end
def remove_local_env_file
deployer_exec("rm .env")
end
def assert_remote_env_file(contents)
assert_equal contents, docker_compose("exec vm1 cat /root/.kamal/env/roles/app-web.env", capture: true)
def assert_remote_env_file(contents, vm:)
assert_equal contents, docker_compose("exec #{vm} cat /root/.kamal/env/roles/app-web.env", capture: true)
end
def assert_no_remote_env_file