From 49ce64de87564182525bc67d1c77f729f265bdb4 Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Wed, 6 Mar 2024 09:39:30 +0000 Subject: [PATCH] Add push_env config This setting allows you to automatically push env files when deploying. The default is not to push any files, but you can set it to `all`, `clear` or `secret` to push the relevant files. The most useful setting is `clear` which will push the clear env files every time you deploy. In addition you can choose the env_type to push when calling `kamal env push` directly: ``` kamal env push --env-type clear kamal env push --env-type secret kamal env push --env-type all # same as kamal env push ``` --- lib/kamal/cli/env.rb | 16 ++- lib/kamal/cli/main.rb | 13 +++ lib/kamal/configuration.rb | 16 ++- test/cli/main_test.rb | 104 +++++++++++++----- test/configuration_test.rb | 9 ++ test/fixtures/deploy_push_clear_env.yml | 37 +++++++ .../docker/deployer/app/config/deploy.yml | 1 + test/integration/main_test.rb | 6 + 8 files changed, 168 insertions(+), 34 deletions(-) create mode 100644 test/fixtures/deploy_push_clear_env.yml diff --git a/lib/kamal/cli/env.rb b/lib/kamal/cli/env.rb index aeaa21a1..75cdd19b 100644 --- a/lib/kamal/cli/env.rb +++ b/lib/kamal/cli/env.rb @@ -2,7 +2,11 @@ require "tempfile" class Kamal::Cli::Env < Kamal::Cli::Base desc "push", "Push the env files to the remote hosts" + option :env_type, type: :string, desc: "Type of env files", enum: %w[secret clear all], default: "all" def push + secret = %w[secret all].include?(options[:env_type]) + clear = %w[clear all].include?(options[:env_type]) + mutating do on(KAMAL.hosts) do execute *KAMAL.auditor.record("Pushed env files"), verbosity: :debug @@ -10,23 +14,23 @@ class Kamal::Cli::Env < Kamal::Cli::Base KAMAL.roles_on(host).each do |role| role_config = KAMAL.config.role(role) execute *KAMAL.app(role: role).make_env_directory - upload! StringIO.new(role_config.env_file.secret), role_config.host_secret_env_file_path, mode: 400 - upload! StringIO.new(role_config.env_file.clear), role_config.host_clear_env_file_path, mode: 400 + upload! StringIO.new(role_config.env_file.secret), role_config.host_secret_env_file_path, mode: 400 if secret + upload! StringIO.new(role_config.env_file.clear), role_config.host_clear_env_file_path, mode: 400 if clear end end on(KAMAL.traefik_hosts) do execute *KAMAL.traefik.make_env_directory - upload! StringIO.new(KAMAL.traefik.env_file.secret), KAMAL.traefik.host_secret_env_file_path, mode: 400 - upload! StringIO.new(KAMAL.traefik.env_file.clear), KAMAL.traefik.host_clear_env_file_path, mode: 400 + upload! StringIO.new(KAMAL.traefik.env_file.secret), KAMAL.traefik.host_secret_env_file_path, mode: 400 if secret + upload! StringIO.new(KAMAL.traefik.env_file.clear), KAMAL.traefik.host_clear_env_file_path, mode: 400 if clear end on(KAMAL.accessory_hosts) do KAMAL.accessories_on(host).each do |accessory| accessory_config = KAMAL.config.accessory(accessory) execute *KAMAL.accessory(accessory).make_env_directory - upload! StringIO.new(accessory_config.env_file.secret), accessory_config.host_secret_env_file_path, mode: 400 - upload! StringIO.new(accessory_config.env_file.clear), accessory_config.host_clear_env_file_path, mode: 400 + upload! StringIO.new(accessory_config.env_file.secret), accessory_config.host_secret_env_file_path, mode: 400 if secret + upload! StringIO.new(accessory_config.env_file.clear), accessory_config.host_clear_env_file_path, mode: 400 if clear end end end diff --git a/lib/kamal/cli/main.rb b/lib/kamal/cli/main.rb index 6adc36fe..9ac708b7 100644 --- a/lib/kamal/cli/main.rb +++ b/lib/kamal/cli/main.rb @@ -35,6 +35,8 @@ class Kamal::Cli::Main < Kamal::Cli::Base run_hook "pre-deploy" + push_env(invoke_options) + say "Ensure Traefik is running...", :magenta invoke "kamal:cli:traefik:boot", [], invoke_options @@ -73,6 +75,8 @@ class Kamal::Cli::Main < Kamal::Cli::Base run_hook "pre-deploy" + push_env(invoke_options) + say "Ensure app can pass healthcheck...", :magenta invoke "kamal:cli:healthcheck:perform", [], invoke_options @@ -99,6 +103,8 @@ class Kamal::Cli::Main < Kamal::Cli::Base if container_available?(version) run_hook "pre-deploy" + push_env(invoke_options) + invoke "kamal:cli:app:boot", [], invoke_options.merge(version: version) rolled_back = true else @@ -262,4 +268,11 @@ class Kamal::Cli::Main < Kamal::Cli::Base def deploy_options { "version" => KAMAL.config.version }.merge(options.without("skip_push")) end + + def push_env(invoke_options) + if KAMAL.config.push_env + say "Pushing #{KAMAL.config.push_env} env files..." + invoke "kamal:cli:env:push", [], invoke_options.merge(env_type: KAMAL.config.push_env) + end + end end diff --git a/lib/kamal/configuration.rb b/lib/kamal/configuration.rb index a3b074d5..f31a3035 100644 --- a/lib/kamal/configuration.rb +++ b/lib/kamal/configuration.rb @@ -6,7 +6,7 @@ require "erb" require "net/ssh/proxy/jump" class Kamal::Configuration - delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true + delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, :push_env, to: :raw_config, allow_nil: true delegate :argumentize, :optionize, to: Kamal::Utils attr_reader :destination, :raw_config @@ -222,7 +222,11 @@ class Kamal::Configuration def valid? - ensure_destination_if_required && ensure_required_keys_present && ensure_valid_kamal_version && ensure_retain_containers_valid + ensure_destination_if_required \ + && ensure_required_keys_present \ + && ensure_valid_kamal_version \ + && ensure_retain_containers_valid \ + && ensure_push_env_valid end def to_h @@ -301,6 +305,14 @@ class Kamal::Configuration true end + def ensure_push_env_valid + if raw_config.push_env && !%w[ all clear secret ].include?(raw_config.push_env) + raise ArgumentError, "push_env must be one of `all`, `clear` `secret`" + end + + true + end + def role_names raw_config.servers.is_a?(Array) ? [ "web" ] : raw_config.servers.keys.sort diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index 8b152d52..7497ba3c 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -122,12 +122,12 @@ class CliMainTest < CliTestCase refute_match /Running the post-deploy hook.../, output end end - + test "deploy without healthcheck if primary host doesn't have traefik" do invoke_options = { "config_file" => "test/fixtures/deploy_workers_only.yml", "version" => "999", "skip_hooks" => false } Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options).never - + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options) Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options) Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options) @@ -152,6 +152,26 @@ class CliMainTest < CliTestCase run_command("deploy", config_file: "deploy_with_secrets") end + test "deploy with push_env" do + invoke_options = { "config_file" => "test/fixtures/deploy_push_clear_env.yml", "version" => "999", "skip_hooks" => false } + + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:registry:login", [], invoke_options) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options.merge(env_type: "clear")) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:boot", [], invoke_options) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:prune:all", [], invoke_options) + + Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) + hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "deploy" } + + run_command("deploy", config_file: "deploy_push_clear_env").tap do |output| + assert_match /Pushing clear env files.../, output + end + end + test "redeploy" do invoke_options = { "config_file" => "test/fixtures/deploy_simple.yml", "version" => "999", "skip_hooks" => false } @@ -188,6 +208,23 @@ class CliMainTest < CliTestCase end end + test "redeploy with push_env" do + invoke_options = { "config_file" => "test/fixtures/deploy_push_clear_env.yml", "version" => "999", "skip_hooks" => false } + + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:build:deliver", [], invoke_options) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:env:push", [], invoke_options.merge(env_type: "clear")) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:healthcheck:perform", [], invoke_options) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)) + Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:boot", [], invoke_options) + + Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) + hook_variables = { version: 999, service_version: "app@999", hosts: "1.1.1.1,1.1.1.2", command: "redeploy" } + + run_command("redeploy", config_file: "deploy_push_clear_env").tap do |output| + assert_match /Pushing clear env files.../, output + end + end + test "rollback bad version" do Thread.report_on_exception = false @@ -200,31 +237,8 @@ class CliMainTest < CliTestCase end test "rollback good version" do - Object.any_instance.stubs(:sleep) - [ "web", "workers" ].each do |role| - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false) - .returns("").at_least_once - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet") - .returns("version-to-rollback\n").at_least_once - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-#{role}-}; done", raise_on_non_zero_exit: false) - .returns("version-to-rollback\n").at_least_once - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") - .returns("running").at_least_once # health check - end + stub_good_rollback - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false) - .returns("corddirectory").at_least_once # health check - - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-version-to-rollback$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") - .returns("unhealthy").at_least_once # health check - - Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) hook_variables = { version: 123, service_version: "app@123", hosts: "1.1.1.1,1.1.1.2,1.1.1.3,1.1.1.4", command: "rollback" } run_command("rollback", "123", config_file: "deploy_with_accessories").tap do |output| @@ -257,6 +271,16 @@ class CliMainTest < CliTestCase end end + test "rollback with push_env" do + invoke_options = { "config_file" => "test/fixtures/deploy_push_clear_env.yml", "version" => "999", "skip_hooks" => false } + + stub_good_rollback + + run_command("rollback", "123", config_file: "deploy_push_clear_env").tap do |output| + assert_match /Pushing clear env files.../, output + end + end + test "details" do Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:traefik:details") Kamal::Cli::Main.any_instance.expects(:invoke).with("kamal:cli:app:details") @@ -454,4 +478,32 @@ class CliMainTest < CliTestCase def run_command(*command, config_file: "deploy_simple") stdouted { Kamal::Cli::Main.start([*command, "-c", "test/fixtures/#{config_file}.yml"]) } end + + def stub_good_rollback + Object.any_instance.stubs(:sleep) + [ "web", "workers" ].each do |role| + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", raise_on_non_zero_exit: false) + .returns("").at_least_once + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet") + .returns("version-to-rollback\n").at_least_once + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :ps, "--filter", "label=service=app", "--filter", "label=role=#{role}", "--filter", "status=running", "--filter", "status=restarting", "--latest", "--format", "\"{{.Names}}\"", "|", "while read line; do echo ${line#app-#{role}-}; done", raise_on_non_zero_exit: false) + .returns("version-to-rollback\n").at_least_once + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-#{role}-123$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("running").at_least_once # health check + end + + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :inspect, "-f '{{ range .Mounts }}{{printf \"%s %s\\n\" .Source .Destination}}{{ end }}'", "app-web-version-to-rollback", "|", :awk, "'$2 == \"/tmp/kamal-cord\" {print $1}'", :raise_on_non_zero_exit => false) + .returns("corddirectory").at_least_once # health check + + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "name=^app-web-version-to-rollback$", "--quiet", "|", :xargs, :docker, :inspect, "--format", "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'") + .returns("unhealthy").at_least_once # health check + + Kamal::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) + end end diff --git a/test/configuration_test.rb b/test/configuration_test.rb index fbc87a91..93ed487c 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -317,4 +317,13 @@ class ConfigurationTest < ActiveSupport::TestCase assert_raises(ArgumentError) { Kamal::Configuration.new(@deploy_with_roles.merge(retain_containers: 0)) } end + + test "push_env" do + assert_nil @config.push_env + assert_equal "all", Kamal::Configuration.new(@deploy.merge(push_env: "all")).push_env + assert_equal "clear", Kamal::Configuration.new(@deploy.merge(push_env: "clear")).push_env + assert_equal "secret", Kamal::Configuration.new(@deploy.merge(push_env: "secret")).push_env + + assert_raises(ArgumentError) { Kamal::Configuration.new(@deploy_with_roles.merge(push_env: "foo")) } + end end diff --git a/test/fixtures/deploy_push_clear_env.yml b/test/fixtures/deploy_push_clear_env.yml new file mode 100644 index 00000000..968d2266 --- /dev/null +++ b/test/fixtures/deploy_push_clear_env.yml @@ -0,0 +1,37 @@ +service: app +image: dhh/app +servers: + web: + - "1.1.1.1" + - "1.1.1.2" + workers: + - "1.1.1.3" + - "1.1.1.4" +registry: + username: user + password: pw +push_env: clear + +accessories: + mysql: + image: mysql:5.7 + host: 1.1.1.3 + port: 3306 + env: + clear: + MYSQL_ROOT_HOST: '%' + secret: + - MYSQL_ROOT_PASSWORD + files: + - test/fixtures/files/my.cnf:/etc/mysql/my.cnf + directories: + - data:/var/lib/mysql + redis: + image: redis:latest + roles: + - web + port: 6379 + directories: + - data:/data + +readiness_delay: 0 diff --git a/test/integration/docker/deployer/app/config/deploy.yml b/test/integration/docker/deployer/app/config/deploy.yml index baedca99..d13435a4 100644 --- a/test/integration/docker/deployer/app/config/deploy.yml +++ b/test/integration/docker/deployer/app/config/deploy.yml @@ -33,3 +33,4 @@ accessories: roles: - web stop_wait_time: 1 +push_env: clear diff --git a/test/integration/main_test.rb b/test/integration/main_test.rb index 26ab8c23..501bdde8 100644 --- a/test/integration/main_test.rb +++ b/test/integration/main_test.rb @@ -7,6 +7,7 @@ class MainTest < IntegrationTest assert_remote_env_file "CLEAR_TOKEN=4321", :clear assert_remote_env_file "SECRET_TOKEN=1234", :secret remove_local_env_file + remove_remote_env_file :clear first_version = latest_app_version @@ -15,6 +16,7 @@ class MainTest < IntegrationTest kamal :deploy assert_app_is_up version: first_version assert_hooks_ran "pre-connect", "pre-build", "pre-deploy", "post-deploy" + assert_remote_env_file "CLEAR_TOKEN=4321", :clear second_version = update_app_rev @@ -70,6 +72,10 @@ class MainTest < IntegrationTest deployer_exec("rm .env") end + def remove_remote_env_file(env_type) + docker_compose("exec vm1 cat /root/.kamal/env/roles/app-web-#{env_type}.env") + end + def assert_remote_env_file(contents, env_type) assert_equal contents, docker_compose("exec vm1 cat /root/.kamal/env/roles/app-web-#{env_type}.env", capture: true) end